From 7ed17ed9e79b9062374fa77feee9471fe7c365b2 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 14:44:59 -0400 Subject: [PATCH 001/110] fixup! Enable the built-in FSMonitor as an experimental feature This reverts commit 3bcb1a2f29a0853933018a75c5f3a14465b433c9. Signed-off-by: Jeff Hostetler --- repo-settings.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/repo-settings.c b/repo-settings.c index fd3a9e04c491e8..08c60a927473e2 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -2,14 +2,13 @@ #include "config.h" #include "repository.h" #include "midx.h" -#include "fsmonitor-ipc.h" -#include "fsmonitor-settings.h" +#include "compat/fsmonitor/fsm-listen.h" #define UPDATE_DEFAULT_BOOL(s,v) do { if (s == -1) { s = v; } } while(0) void prepare_repo_settings(struct repository *r) { - int value, feature_many_files = 0; + int value; char *strval; if (r->settings.initialized) @@ -63,7 +62,6 @@ void prepare_repo_settings(struct repository *r) UPDATE_DEFAULT_BOOL(r->settings.core_multi_pack_index, 1); if (!repo_config_get_bool(r, "feature.manyfiles", &value) && value) { - feature_many_files = 1; UPDATE_DEFAULT_BOOL(r->settings.index_version, 4); UPDATE_DEFAULT_BOOL(r->settings.core_untracked_cache, UNTRACKED_CACHE_WRITE); } @@ -72,11 +70,8 @@ void prepare_repo_settings(struct repository *r) r->settings.fetch_write_commit_graph = value; UPDATE_DEFAULT_BOOL(r->settings.fetch_write_commit_graph, 0); - if (!repo_config_get_bool(r, "feature.experimental", &value) && value) { + if (!repo_config_get_bool(r, "feature.experimental", &value) && value) UPDATE_DEFAULT_BOOL(r->settings.fetch_negotiation_algorithm, FETCH_NEGOTIATION_SKIPPING); - if (feature_many_files && fsmonitor_ipc__is_supported()) - fsm_settings__set_ipc(r); - } /* Hack for test programs like test-dump-untracked-cache */ if (ignore_untracked_cache_config) From 49e51c030a58e947e9d85306aac1a5d27699ae60 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 14:45:37 -0400 Subject: [PATCH 002/110] fixup! fsmonitor: mark the built-in FSMonitor as experimental This reverts commit 63d2d7bb4a2fbe6c9ba2d2164b396b5977cb4c3c. Signed-off-by: Jeff Hostetler --- Documentation/config/core.txt | 4 ++-- Documentation/git-fsmonitor--daemon.txt | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 2db16334252271..323f966a09e6fd 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -71,7 +71,7 @@ git by avoiding unnecessary scanning of files that have not changed. + See the "fsmonitor-watchman" section of linkgit:githooks[5]. + -Note: The value of this config setting is ignored if the (experimental) +Note: The value of this config setting is ignored if the built-in file system monitor is enabled (see `core.useBuiltinFSMonitor`). core.fsmonitorHookVersion:: @@ -91,7 +91,7 @@ Note: The value of this config setting is ignored if the built-in file system monitor is enabled (see `core.useBuiltinFSMonitor`). core.useBuiltinFSMonitor:: - (EXPERIMENTAL) If set to true, enable the built-in file system monitor + If set to true, enable the built-in file system monitor daemon for this working directory (linkgit:git-fsmonitor--daemon[1]). + Like hook-based file system monitors, the built-in file system monitor diff --git a/Documentation/git-fsmonitor--daemon.txt b/Documentation/git-fsmonitor--daemon.txt index cf1d4556a51425..154e7684daae07 100644 --- a/Documentation/git-fsmonitor--daemon.txt +++ b/Documentation/git-fsmonitor--daemon.txt @@ -3,7 +3,7 @@ git-fsmonitor--daemon(1) NAME ---- -git-fsmonitor--daemon - (EXPERIMENTAL) A Built-in File System Monitor +git-fsmonitor--daemon - A Built-in File System Monitor SYNOPSIS -------- @@ -16,9 +16,6 @@ SYNOPSIS DESCRIPTION ----------- -NOTE! This command is still only an experiment, subject to change dramatically -(or even to be abandoned). - A daemon to watch the working directory for file and directory changes using platform-specific file system notification facilities. From bfa0044eccdae872c740f88a96d949d199e9b1b9 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:09:11 -0400 Subject: [PATCH 003/110] fixup! mingw: support long paths Revert the FSMonitor-related portions of 64da6f21825811a3bd8542f8617d950f0099b18c Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-listen-win32.c | 4 ++-- compat/fsmonitor/fsm-settings-win32.c | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/compat/fsmonitor/fsm-listen-win32.c b/compat/fsmonitor/fsm-listen-win32.c index 9bb92935595151..2431492c3a1c07 100644 --- a/compat/fsmonitor/fsm-listen-win32.c +++ b/compat/fsmonitor/fsm-listen-win32.c @@ -113,9 +113,9 @@ static struct one_watch *create_watch(struct fsmonitor_daemon_state *state, DWORD share_mode = FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE; HANDLE hDir; - wchar_t wpath[MAX_LONG_PATH]; + wchar_t wpath[MAX_PATH]; - if (xutftowcs_long_path(wpath, path) < 0) { + if (xutftowcs_path(wpath, path) < 0) { error(_("could not convert to wide characters: '%s'"), path); return NULL; } diff --git a/compat/fsmonitor/fsm-settings-win32.c b/compat/fsmonitor/fsm-settings-win32.c index 83d91f85ece3fa..39e907046f14d7 100644 --- a/compat/fsmonitor/fsm-settings-win32.c +++ b/compat/fsmonitor/fsm-settings-win32.c @@ -76,8 +76,8 @@ static enum fsmonitor_reason is_virtual(struct repository *r) */ static enum fsmonitor_reason is_remote(struct repository *r) { - wchar_t wpath[MAX_LONG_PATH]; - wchar_t wfullpath[MAX_LONG_PATH]; + wchar_t wpath[MAX_PATH]; + wchar_t wfullpath[MAX_PATH]; size_t wlen; UINT driveType; @@ -85,7 +85,7 @@ static enum fsmonitor_reason is_remote(struct repository *r) * Do everything in wide chars because the drive letter might be * a multi-byte sequence. See win32_has_dos_drive_prefix(). */ - if (xutftowcs_long_path(wpath, r->worktree) < 0) + if (xutftowcs_path(wpath, r->worktree) < 0) return FSMONITOR_REASON_ZERO; /* @@ -103,7 +103,7 @@ static enum fsmonitor_reason is_remote(struct repository *r) * slashes to backslashes. This is essential to get GetDriveTypeW() * correctly handle some UNC "\\server\share\..." paths. */ - if (!GetFullPathNameW(wpath, MAX_LONG_PATH, wfullpath, NULL)) + if (!GetFullPathNameW(wpath, MAX_PATH, wfullpath, NULL)) return FSMONITOR_REASON_ZERO; driveType = GetDriveTypeW(wfullpath); From 98b55c3021b0e13c3a973821aaff1a2f5f7c1496 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:13:54 -0400 Subject: [PATCH 004/110] fixup! fsmonitor-settings: remote repos on Windows are incompatible with FSMonitor This reverts commit a002e816f2af54129a6283c41a8bd6f0b87cc528. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-settings-win32.c | 102 -------------------------- 1 file changed, 102 deletions(-) diff --git a/compat/fsmonitor/fsm-settings-win32.c b/compat/fsmonitor/fsm-settings-win32.c index 39e907046f14d7..7caa79570afa92 100644 --- a/compat/fsmonitor/fsm-settings-win32.c +++ b/compat/fsmonitor/fsm-settings-win32.c @@ -2,7 +2,6 @@ #include "config.h" #include "repository.h" #include "fsmonitor-settings.h" -#include "fsmonitor.h" /* * GVFS (aka VFS for Git) is incompatible with FSMonitor. @@ -24,103 +23,6 @@ static enum fsmonitor_reason is_virtual(struct repository *r) return FSMONITOR_REASON_ZERO; } -/* - * Remote working directories are problematic for FSMonitor. - * - * The underlying file system on the server machine and/or the remote - * mount type dictates whether notification events are available at - * all to remote client machines. - * - * Kernel differences between the server and client machines also - * dictate the how (buffering, frequency, de-dup) the events are - * delivered to client machine processes. - * - * A client machine (such as a laptop) may choose to suspend/resume - * and it is unclear (without lots of testing) whether the watcher can - * resync after a resume. We might be able to treat this as a normal - * "events were dropped by the kernel" event and do our normal "flush - * and resync" --or-- we might need to close the existing (zombie?) - * notification fd and create a new one. - * - * In theory, the above issues need to be addressed whether we are - * using the Hook or IPC API. - * - * So (for now at least), mark remote working directories as - * incompatible. - * - * Notes for testing: - * - * (a) Windows allows a network share to be mapped to a drive letter. - * (This is the normal method to access it.) - * - * $ NET USE Z: \\server\share - * $ git -C Z:/repo status - * - * (b) Windows allows a network share to be referenced WITHOUT mapping - * it to drive letter. - * - * $ NET USE \\server\share\dir - * $ git -C //server/share/repo status - * - * (c) Windows allows "SUBST" to create a fake drive mapping to an - * arbitrary path (which may be remote) - * - * $ SUBST Q: Z:\repo - * $ git -C Q:/ status - * - * (d) Windows allows a directory symlink to be created on a local - * file system that points to a remote repo. - * - * $ mklink /d ./link //server/share/repo - * $ git -C ./link status - */ -static enum fsmonitor_reason is_remote(struct repository *r) -{ - wchar_t wpath[MAX_PATH]; - wchar_t wfullpath[MAX_PATH]; - size_t wlen; - UINT driveType; - - /* - * Do everything in wide chars because the drive letter might be - * a multi-byte sequence. See win32_has_dos_drive_prefix(). - */ - if (xutftowcs_path(wpath, r->worktree) < 0) - return FSMONITOR_REASON_ZERO; - - /* - * GetDriveTypeW() requires a final slash. We assume that the - * worktree pathname points to an actual directory. - */ - wlen = wcslen(wpath); - if (wpath[wlen - 1] != L'\\' && wpath[wlen - 1] != L'/') { - wpath[wlen++] = L'\\'; - wpath[wlen] = 0; - } - - /* - * Normalize the path. If nothing else, this converts forward - * slashes to backslashes. This is essential to get GetDriveTypeW() - * correctly handle some UNC "\\server\share\..." paths. - */ - if (!GetFullPathNameW(wpath, MAX_PATH, wfullpath, NULL)) - return FSMONITOR_REASON_ZERO; - - driveType = GetDriveTypeW(wfullpath); - trace_printf_key(&trace_fsmonitor, - "DriveType '%s' L'%S' (%u)", - r->worktree, wfullpath, driveType); - - if (driveType == DRIVE_REMOTE) { - trace_printf_key(&trace_fsmonitor, - "is_remote('%s') true", - r->worktree); - return FSMONITOR_REASON_REMOTE; - } - - return FSMONITOR_REASON_ZERO; -} - enum fsmonitor_reason fsm_os__incompatible(struct repository *r) { enum fsmonitor_reason reason; @@ -129,9 +31,5 @@ enum fsmonitor_reason fsm_os__incompatible(struct repository *r) if (reason) return reason; - reason = is_remote(r); - if (reason) - return reason; - return FSMONITOR_REASON_ZERO; } From 479df9ff685964d8151d8b12245f273076176370 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:14:07 -0400 Subject: [PATCH 005/110] fixup! fsmonitor-settings: remote repos on MacOS are incompatible with FSMonitor This reverts commit da396b62982ffadb98525d4f155e4563bdf41aea. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-settings-darwin.c | 66 -------------------------- fsmonitor-settings.c | 5 -- fsmonitor-settings.h | 1 - 3 files changed, 72 deletions(-) diff --git a/compat/fsmonitor/fsm-settings-darwin.c b/compat/fsmonitor/fsm-settings-darwin.c index c3b719d6fb0977..176a6f5726c729 100644 --- a/compat/fsmonitor/fsm-settings-darwin.c +++ b/compat/fsmonitor/fsm-settings-darwin.c @@ -2,74 +2,8 @@ #include "config.h" #include "repository.h" #include "fsmonitor-settings.h" -#include "fsmonitor.h" -#include -#include - -/* - * Remote working directories are problematic for FSMonitor. - * - * The underlying file system on the server machine and/or the remote - * mount type (NFS, SAMBA, etc.) dictates whether notification events - * are available at all to remote client machines. - * - * Kernel differences between the server and client machines also - * dictate the how (buffering, frequency, de-dup) the events are - * delivered to client machine processes. - * - * A client machine (such as a laptop) may choose to suspend/resume - * and it is unclear (without lots of testing) whether the watcher can - * resync after a resume. We might be able to treat this as a normal - * "events were dropped by the kernel" event and do our normal "flush - * and resync" --or-- we might need to close the existing (zombie?) - * notification fd and create a new one. - * - * In theory, the above issues need to be addressed whether we are - * using the Hook or IPC API. - * - * For the builtin FSMonitor, we create the Unix domain socket for the - * IPC in the .git directory. If the working directory is remote, - * then the socket will be created on the remote file system. This - * can fail if the remote file system does not support UDS file types - * (e.g. smbfs to a Windows server) or if the remote kernel does not - * allow a non-local process to bind() the socket. (These problems - * could be fixed by moving the UDS out of the .git directory and to a - * well-known local directory on the client machine, but care should - * be taken to ensure that $HOME is actually local and not a managed - * file share.) - * - * So (for now at least), mark remote working directories as - * incompatible. - */ -static enum fsmonitor_reason is_remote(struct repository *r) -{ - struct statfs fs; - - if (statfs(r->worktree, &fs) == -1) { - int saved_errno = errno; - trace_printf_key(&trace_fsmonitor, "statfs('%s') failed: %s", - r->worktree, strerror(saved_errno)); - errno = saved_errno; - return FSMONITOR_REASON_ZERO; - } - - trace_printf_key(&trace_fsmonitor, - "statfs('%s') [type 0x%08x][flags 0x%08x] '%s'", - r->worktree, fs.f_type, fs.f_flags, fs.f_fstypename); - - if (!(fs.f_flags & MNT_LOCAL)) - return FSMONITOR_REASON_REMOTE; - - return FSMONITOR_REASON_ZERO; -} enum fsmonitor_reason fsm_os__incompatible(struct repository *r) { - enum fsmonitor_reason reason; - - reason = is_remote(r); - if (reason) - return reason; - return FSMONITOR_REASON_ZERO; } diff --git a/fsmonitor-settings.c b/fsmonitor-settings.c index 1aa77b3193e1ff..41cfeea9e85a7a 100644 --- a/fsmonitor-settings.c +++ b/fsmonitor-settings.c @@ -160,11 +160,6 @@ static void create_reason_message(struct repository *r, _("virtual repos are incompatible with fsmonitor")); return; - case FSMONITOR_REASON_REMOTE: - strbuf_addstr(buf_reason, - _("remote repos are incompatible with fsmonitor")); - return; - default: BUG("Unhandled case in create_reason_message '%d'", s->reason); } diff --git a/fsmonitor-settings.h b/fsmonitor-settings.h index da203781725c1f..7c8da4e80a3060 100644 --- a/fsmonitor-settings.h +++ b/fsmonitor-settings.h @@ -17,7 +17,6 @@ enum fsmonitor_reason { FSMONITOR_REASON_ZERO = 0, FSMONITOR_REASON_BARE = 1, FSMONITOR_REASON_VIRTUAL = 2, - FSMONITOR_REASON_REMOTE = 3, }; void fsm_settings__set_ipc(struct repository *r); From 627671214f2c035a2e0159c2d5a0809cc504d02c Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:14:11 -0400 Subject: [PATCH 006/110] fixup! fsmonitor-settings: stub in platform-specific incompatibility checking on MacOS This reverts commit 7b8c024f511a7240da3d83017019b284802c90d7. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-settings-darwin.c | 9 --------- config.mak.uname | 1 - contrib/buildsystems/CMakeLists.txt | 3 --- 3 files changed, 13 deletions(-) delete mode 100644 compat/fsmonitor/fsm-settings-darwin.c diff --git a/compat/fsmonitor/fsm-settings-darwin.c b/compat/fsmonitor/fsm-settings-darwin.c deleted file mode 100644 index 176a6f5726c729..00000000000000 --- a/compat/fsmonitor/fsm-settings-darwin.c +++ /dev/null @@ -1,9 +0,0 @@ -#include "cache.h" -#include "config.h" -#include "repository.h" -#include "fsmonitor-settings.h" - -enum fsmonitor_reason fsm_os__incompatible(struct repository *r) -{ - return FSMONITOR_REASON_ZERO; -} diff --git a/config.mak.uname b/config.mak.uname index 3aa2e00d7067a8..f5935227e22f55 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -148,7 +148,6 @@ ifeq ($(uname_S),Darwin) endif endif FSMONITOR_DAEMON_BACKEND = darwin - FSMONITOR_OS_SETTINGS = darwin BASIC_LDFLAGS += -framework CoreServices endif ifeq ($(uname_S),SunOS) diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index a1f5a905642b41..8bb57ead5a0e22 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -303,9 +303,6 @@ endif() if(CMAKE_SYSTEM_NAME STREQUAL "Windows") add_compile_definitions(HAVE_FSMONITOR_OS_SETTINGS) list(APPEND compat_SOURCES compat/fsmonitor/fsm-settings-win32.c) -elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") - add_compile_definitions(HAVE_FSMONITOR_OS_SETTINGS) - list(APPEND compat_SOURCES compat/fsmonitor/fsm-settings-darwin.c) endif() set(EXE_EXTENSION ${CMAKE_EXECUTABLE_SUFFIX}) From 1a545a1fa03f00a95c363f4fbb413bc804c70a10 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:14:13 -0400 Subject: [PATCH 007/110] fixup! fsmonitor--daemon: background daemon must free the console on windows This reverts commit e3fc6efce3848c00fc2771f12b0800d8f442507b. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index d9a9b8e6b832ad..a4e3f6e3099ff3 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -1302,7 +1302,7 @@ static int fsmonitor_run_daemon(void) return err; } -static int try_to_run_foreground_daemon(int free_console) +static int try_to_run_foreground_daemon(void) { /* * Technically, we don't need to probe for an existing daemon @@ -1320,11 +1320,6 @@ static int try_to_run_foreground_daemon(int free_console) the_repository->worktree); fflush(stdout); -#ifdef GIT_WINDOWS_NATIVE - if (free_console) - FreeConsole(); -#endif - return !!fsmonitor_run_daemon(); } @@ -1356,7 +1351,6 @@ static int spawn_fsmonitor(pid_t *pid) strvec_push(&args, git_exe); strvec_push(&args, "fsmonitor--daemon"); strvec_push(&args, "run"); - strvec_push(&args, "--free-console"); strvec_pushf(&args, "--ipc-threads=%d", fsmonitor__ipc_threads); *pid = mingw_spawnvpe(args.v[0], args.v, NULL, NULL, in, out, out); @@ -1520,10 +1514,8 @@ static int try_to_start_background_daemon(void) int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) { const char *subcmd; - int free_console = 0; struct option options[] = { - OPT_BOOL(0, "free-console", &free_console, N_("free console")), OPT_INTEGER(0, "ipc-threads", &fsmonitor__ipc_threads, N_("use ipc worker threads")), @@ -1567,7 +1559,7 @@ int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) return !!try_to_start_background_daemon(); if (!strcmp(subcmd, "run")) - return !!try_to_run_foreground_daemon(free_console); + return !!try_to_run_foreground_daemon(); if (!strcmp(subcmd, "stop")) return !!do_as_client__send_stop(); From 5a54dfa08ab8cd7a3212f221a90b15ea9ec3c475 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:14:15 -0400 Subject: [PATCH 008/110] fixup! fsmonitor-settings: virtual repos are incompatible with FSMonitor This reverts commit 7b37f49707b28fce7e873c2d232b1d5bb82c0faa. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-settings-win32.c | 26 -------------------------- fsmonitor-settings.c | 5 ----- fsmonitor-settings.h | 1 - t/t7519-status-fsmonitor.sh | 9 --------- 4 files changed, 41 deletions(-) diff --git a/compat/fsmonitor/fsm-settings-win32.c b/compat/fsmonitor/fsm-settings-win32.c index 7caa79570afa92..176a6f5726c729 100644 --- a/compat/fsmonitor/fsm-settings-win32.c +++ b/compat/fsmonitor/fsm-settings-win32.c @@ -3,33 +3,7 @@ #include "repository.h" #include "fsmonitor-settings.h" -/* - * GVFS (aka VFS for Git) is incompatible with FSMonitor. - * - * Granted, core Git does not know anything about GVFS and we - * shouldn't make assumptions about a downstream feature, but users - * can install both versions. And this can lead to incorrect results - * from core Git commands. So, without bringing in any of the GVFS - * code, do a simple config test for a published config setting. (We - * do not look at the various *_TEST_* environment variables.) - */ -static enum fsmonitor_reason is_virtual(struct repository *r) -{ - const char *const_str; - - if (!repo_config_get_value(r, "core.virtualfilesystem", &const_str)) - return FSMONITOR_REASON_VIRTUAL; - - return FSMONITOR_REASON_ZERO; -} - enum fsmonitor_reason fsm_os__incompatible(struct repository *r) { - enum fsmonitor_reason reason; - - reason = is_virtual(r); - if (reason) - return reason; - return FSMONITOR_REASON_ZERO; } diff --git a/fsmonitor-settings.c b/fsmonitor-settings.c index 41cfeea9e85a7a..408c3f159d6ed6 100644 --- a/fsmonitor-settings.c +++ b/fsmonitor-settings.c @@ -155,11 +155,6 @@ static void create_reason_message(struct repository *r, _("bare repos are incompatible with fsmonitor")); return; - case FSMONITOR_REASON_VIRTUAL: - strbuf_addstr(buf_reason, - _("virtual repos are incompatible with fsmonitor")); - return; - default: BUG("Unhandled case in create_reason_message '%d'", s->reason); } diff --git a/fsmonitor-settings.h b/fsmonitor-settings.h index 7c8da4e80a3060..813a5406197c61 100644 --- a/fsmonitor-settings.h +++ b/fsmonitor-settings.h @@ -16,7 +16,6 @@ enum fsmonitor_mode { enum fsmonitor_reason { FSMONITOR_REASON_ZERO = 0, FSMONITOR_REASON_BARE = 1, - FSMONITOR_REASON_VIRTUAL = 2, }; void fsm_settings__set_ipc(struct repository *r); diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 2867806b5d1194..b9407d4738b129 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -81,15 +81,6 @@ test_expect_success FSMONITOR_DAEMON 'run fsmonitor-daemon in bare repo' ' grep "bare repos are incompatible with fsmonitor" actual ' -test_expect_success MINGW,FSMONITOR_DAEMON 'run fsmonitor-daemon in virtual repo' ' - test_when_finished "rm -rf ./fake-virtual-clone actual" && - git init fake-virtual-clone && - test_must_fail git -C ./fake-virtual-clone \ - -c core.virtualfilesystem=true \ - fsmonitor--daemon run 2>actual && - grep "virtual repos are incompatible with fsmonitor" actual -' - test_expect_success 'setup' ' mkdir -p .git/hooks && : >tracked && From cd4411cb3c093e8a057bfc85636507e3049d5c28 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:14:18 -0400 Subject: [PATCH 009/110] fixup! fsmonitor-settings: stub in platform-specific incompatibility checking This reverts commit 56b6c9aa9926d5c23719a2de680b825a6eb2f0c0. Signed-off-by: Jeff Hostetler --- Makefile | 13 ------------- compat/fsmonitor/fsm-settings-win32.c | 9 --------- config.mak.uname | 2 -- contrib/buildsystems/CMakeLists.txt | 5 ----- fsmonitor-settings.c | 12 ------------ fsmonitor-settings.h | 13 ------------- 6 files changed, 54 deletions(-) delete mode 100644 compat/fsmonitor/fsm-settings-win32.c diff --git a/Makefile b/Makefile index 89567e4b6612f0..dca63dd2d55cc0 100644 --- a/Makefile +++ b/Makefile @@ -476,11 +476,6 @@ all:: # `compat/fsmonitor/fsm-listen-.c` that implements the # `fsm_listen__*()` routines. # -# If your platform has os-specific ways to tell if a repo is incompatible with -# fsmonitor (whether the hook or ipc daemon version), set FSMONITOR_OS_SETTINGS -# to the "" of the corresponding `compat/fsmonitor/fsm-settings-.c` -# that implements the `fsm_os_settings__*()` routines. -# # Define DEVELOPER to enable more compiler warnings. Compiler version # and family are auto detected, but could be overridden by defining # COMPILER_FEATURES (see config.mak.dev). You can still set @@ -1956,11 +1951,6 @@ ifdef FSMONITOR_DAEMON_BACKEND COMPAT_OBJS += compat/fsmonitor/fsm-listen-$(FSMONITOR_DAEMON_BACKEND).o endif -ifdef FSMONITOR_OS_SETTINGS - COMPAT_CFLAGS += -DHAVE_FSMONITOR_OS_SETTINGS - COMPAT_OBJS += compat/fsmonitor/fsm-settings-$(FSMONITOR_OS_SETTINGS).o -endif - ifeq ($(TCLTK_PATH),) NO_TCLTK = NoThanks endif @@ -2845,9 +2835,6 @@ GIT-BUILD-OPTIONS: FORCE ifdef FSMONITOR_DAEMON_BACKEND @echo FSMONITOR_DAEMON_BACKEND=\''$(subst ','\'',$(subst ','\'',$(FSMONITOR_DAEMON_BACKEND)))'\' >>$@+ endif -ifdef FSMONITOR_OS_SETTINGS - @echo FSMONITOR_OS_SETTINGS=\''$(subst ','\'',$(subst ','\'',$(FSMONITOR_OS_SETTINGS)))'\' >>$@+ -endif ifdef TEST_OUTPUT_DIRECTORY @echo TEST_OUTPUT_DIRECTORY=\''$(subst ','\'',$(subst ','\'',$(TEST_OUTPUT_DIRECTORY)))'\' >>$@+ endif diff --git a/compat/fsmonitor/fsm-settings-win32.c b/compat/fsmonitor/fsm-settings-win32.c deleted file mode 100644 index 176a6f5726c729..00000000000000 --- a/compat/fsmonitor/fsm-settings-win32.c +++ /dev/null @@ -1,9 +0,0 @@ -#include "cache.h" -#include "config.h" -#include "repository.h" -#include "fsmonitor-settings.h" - -enum fsmonitor_reason fsm_os__incompatible(struct repository *r) -{ - return FSMONITOR_REASON_ZERO; -} diff --git a/config.mak.uname b/config.mak.uname index f5935227e22f55..a27ccb583d43f2 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -423,7 +423,6 @@ ifeq ($(uname_S),Windows) # # SNPRINTF_RETURNS_BOGUS = YesPlease FSMONITOR_DAEMON_BACKEND = win32 - FSMONITOR_OS_SETTINGS = win32 NO_SVN_TESTS = YesPlease RUNTIME_PREFIX = YesPlease HAVE_WPGMPTR = YesWeDo @@ -612,7 +611,6 @@ ifneq (,$(findstring MINGW,$(uname_S))) NO_MKDTEMP = YesPlease NO_SVN_TESTS = YesPlease FSMONITOR_DAEMON_BACKEND = win32 - FSMONITOR_OS_SETTINGS = win32 RUNTIME_PREFIX = YesPlease HAVE_WPGMPTR = YesWeDo NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 8bb57ead5a0e22..79e4b0dae384e4 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -300,11 +300,6 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") list(APPEND compat_SOURCES compat/fsmonitor/fsm-listen-darwin.c) endif() -if(CMAKE_SYSTEM_NAME STREQUAL "Windows") - add_compile_definitions(HAVE_FSMONITOR_OS_SETTINGS) - list(APPEND compat_SOURCES compat/fsmonitor/fsm-settings-win32.c) -endif() - set(EXE_EXTENSION ${CMAKE_EXECUTABLE_SUFFIX}) #header checks diff --git a/fsmonitor-settings.c b/fsmonitor-settings.c index 408c3f159d6ed6..5a803a41d57434 100644 --- a/fsmonitor-settings.c +++ b/fsmonitor-settings.c @@ -33,18 +33,6 @@ static int check_for_incompatible(struct repository *r) return 1; } -#ifdef HAVE_FSMONITOR_OS_SETTINGS - { - enum fsmonitor_reason reason; - - reason = fsm_os__incompatible(r); - if (reason != FSMONITOR_REASON_ZERO) { - set_incompatible(r, reason); - return 1; - } - } -#endif - return 0; } diff --git a/fsmonitor-settings.h b/fsmonitor-settings.h index 813a5406197c61..eb45524b1f2297 100644 --- a/fsmonitor-settings.h +++ b/fsmonitor-settings.h @@ -29,17 +29,4 @@ enum fsmonitor_reason fsm_settings__get_reason(struct repository *r, struct fsmonitor_settings; -#ifdef HAVE_FSMONITOR_OS_SETTINGS -/* - * Ask platform-specific code whether the repository is incompatible - * with fsmonitor (both hook and ipc modes). For example, if the working - * directory is on a remote volume and mounted via a technology that does - * not support notification events. - * - * fsm_os__* routines should considered private to fsm_settings__ - * routines. - */ -enum fsmonitor_reason fsm_os__incompatible(struct repository *r); -#endif /* HAVE_FSMONITOR_OS_SETTINGS */ - #endif /* FSMONITOR_SETTINGS_H */ From 09f93852e15c0a52c8f1068ad7cf99378a9078a5 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:14:24 -0400 Subject: [PATCH 010/110] fixup! ipc-win32: add trace2 debugging This reverts commit ddab099c559dd134559bae922248eca54582f98a. Signed-off-by: Jeff Hostetler --- compat/simple-ipc/ipc-win32.c | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/compat/simple-ipc/ipc-win32.c b/compat/simple-ipc/ipc-win32.c index 6c8a308de1351b..8e889d6a506dc0 100644 --- a/compat/simple-ipc/ipc-win32.c +++ b/compat/simple-ipc/ipc-win32.c @@ -49,9 +49,6 @@ static enum ipc_active_state get_active_state(wchar_t *pipe_path) if (GetLastError() == ERROR_FILE_NOT_FOUND) return IPC_STATE__PATH_NOT_FOUND; - trace2_data_intmax("ipc-debug", NULL, "getstate/waitpipe/gle", - (intmax_t)GetLastError()); - return IPC_STATE__OTHER_ERROR; } @@ -115,11 +112,6 @@ static enum ipc_active_state connect_to_server( if (GetLastError() == ERROR_SEM_TIMEOUT) return IPC_STATE__NOT_LISTENING; - gle = GetLastError(); - trace2_data_intmax("ipc-debug", NULL, - "connect/waitpipe/gle", - (intmax_t)gle); - return IPC_STATE__OTHER_ERROR; } @@ -141,31 +133,17 @@ static enum ipc_active_state connect_to_server( break; /* try again */ default: - trace2_data_intmax("ipc-debug", NULL, - "connect/createfile/gle", - (intmax_t)gle); - return IPC_STATE__OTHER_ERROR; } } if (!SetNamedPipeHandleState(hPipe, &mode, NULL, NULL)) { - gle = GetLastError(); - trace2_data_intmax("ipc-debug", NULL, - "connect/setpipestate/gle", - (intmax_t)gle); - CloseHandle(hPipe); return IPC_STATE__OTHER_ERROR; } *pfd = _open_osfhandle((intptr_t)hPipe, O_RDWR|O_BINARY); if (*pfd < 0) { - gle = GetLastError(); - trace2_data_intmax("ipc-debug", NULL, - "connect/openosfhandle/gle", - (intmax_t)gle); - CloseHandle(hPipe); return IPC_STATE__OTHER_ERROR; } From 9fd734e53e481c004ecc9e4d19b0e379f1abf589 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:14:26 -0400 Subject: [PATCH 011/110] fixup! t/helper/fsmonitor-client: create stress test This reverts commit 751d7c39ddba4b60c86d172947991e2d1fdb6d8b. Signed-off-by: Jeff Hostetler --- t/helper/test-fsmonitor-client.c | 105 ------------------------------- 1 file changed, 105 deletions(-) diff --git a/t/helper/test-fsmonitor-client.c b/t/helper/test-fsmonitor-client.c index 9dd2f9af553d34..f7a5b3a32fab86 100644 --- a/t/helper/test-fsmonitor-client.c +++ b/t/helper/test-fsmonitor-client.c @@ -7,8 +7,6 @@ #include "cache.h" #include "parse-options.h" #include "fsmonitor-ipc.h" -#include "thread-utils.h" -#include "trace2.h" #ifndef HAVE_FSMONITOR_DAEMON_BACKEND int cmd__fsmonitor_client(int argc, const char **argv) @@ -81,120 +79,20 @@ static int do_send_flush(void) return 0; } -struct hammer_thread_data -{ - pthread_t pthread_id; - int thread_nr; - - int nr_requests; - const char *token; - - int sum_successful; - int sum_errors; -}; - -static void *hammer_thread_proc(void *_hammer_thread_data) -{ - struct hammer_thread_data *data = _hammer_thread_data; - struct strbuf answer = STRBUF_INIT; - int k; - int ret; - - trace2_thread_start("hammer"); - - for (k = 0; k < data->nr_requests; k++) { - strbuf_reset(&answer); - - ret = fsmonitor_ipc__send_query(data->token, &answer); - if (ret < 0) - data->sum_errors++; - else - data->sum_successful++; - } - - strbuf_release(&answer); - trace2_thread_exit(); - return NULL; -} - -/* - * Start a pool of client threads that will each send a series of - * commands to the daemon. - * - * The goal is to overload the daemon with a sustained series of - * concurrent requests. - */ -static int do_hammer(const char *token, int nr_threads, int nr_requests) -{ - struct hammer_thread_data *data = NULL; - int k; - int sum_join_errors = 0; - int sum_commands = 0; - int sum_errors = 0; - - if (!token || !*token) - token = get_token_from_index(); - if (nr_threads < 1) - nr_threads = 1; - if (nr_requests < 1) - nr_requests = 1; - - CALLOC_ARRAY(data, nr_threads); - - for (k = 0; k < nr_threads; k++) { - struct hammer_thread_data *p = &data[k]; - p->thread_nr = k; - p->nr_requests = nr_requests; - p->token = token; - - if (pthread_create(&p->pthread_id, NULL, hammer_thread_proc, p)) { - warning("failed to create thread[%d] skipping remainder", k); - nr_threads = k; - break; - } - } - - for (k = 0; k < nr_threads; k++) { - struct hammer_thread_data *p = &data[k]; - - if (pthread_join(p->pthread_id, NULL)) - sum_join_errors++; - sum_commands += p->sum_successful; - sum_errors += p->sum_errors; - } - - fprintf(stderr, "HAMMER: [threads %d][requests %d] [ok %d][err %d][join %d]\n", - nr_threads, nr_requests, sum_commands, sum_errors, sum_join_errors); - - free(data); - - /* - * TODO Decide if/when to return an error or call die(). - */ - return 0; -} - int cmd__fsmonitor_client(int argc, const char **argv) { const char *subcmd; const char *token = NULL; - int nr_threads = 1; - int nr_requests = 1; const char * const fsmonitor_client_usage[] = { N_("test-helper fsmonitor-client query []"), N_("test-helper fsmonitor-client flush"), - N_("test-helper fsmonitor-client hammer [] [] []"), NULL, }; struct option options[] = { OPT_STRING(0, "token", &token, N_("token"), N_("command token to send to the server")), - - OPT_INTEGER(0, "threads", &nr_threads, N_("number of client threads")), - OPT_INTEGER(0, "requests", &nr_requests, N_("number of requests per thread")), - OPT_END() }; @@ -218,9 +116,6 @@ int cmd__fsmonitor_client(int argc, const char **argv) if (!strcmp(subcmd, "flush")) return !!do_send_flush(); - if (!strcmp(subcmd, "hammer")) - return !!do_hammer(token, nr_threads, nr_requests); - die("Unhandled subcommand: '%s'", subcmd); } #endif From 126ab52086edf4ba35c0c0d3afa783e60fea3ce5 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:14:28 -0400 Subject: [PATCH 012/110] fixup! t7527: test builtin FSMonitor watching repos with unicode paths This reverts commit 0f46db7c01aff55b566299bad087be7cf13e1947. Signed-off-by: Jeff Hostetler --- t/t7527-builtin-fsmonitor.sh | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index f10a33dde8c270..85e249fda42f6d 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -690,27 +690,4 @@ do done done -# Test Unicode UTF-8 characters in the pathname of the working -# directory. Use of "*A()" routines rather than "*W()" routines -# on Windows can sometimes lead to odd failures. -# -u1=$(printf "u_c3_a6__\xC3\xA6") -u2=$(printf "u_e2_99_ab__\xE2\x99\xAB") -u_values="$u1 $u2" -for u in $u_values -do - test_expect_success "Unicode path: $u" ' - test_when_finished "stop_daemon_delete_repo $u" && - - git init "$u" && - echo 1 >"$u"/file1 && - git -C "$u" add file1 && - git -C "$u" config core.useBuiltinFSMonitor true && - - start_daemon "$u" && - git -C "$u" status >actual && - grep "new file: file1" actual - ' -done - test_done From 238888315a8b9a9a15126bb41183555db144dde7 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:14:31 -0400 Subject: [PATCH 013/110] fixup! t7527: test FS event reporing on MacOS WRT case and Unicode This reverts commit 207eece649a9d4db5778139eefd453e94788569d. Signed-off-by: Jeff Hostetler --- t/t7527-builtin-fsmonitor.sh | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index 85e249fda42f6d..ee94861fe75d77 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -188,36 +188,6 @@ test_expect_success MINGW,SHORTNAMES 'implicit daemon stop (rename GIT~2)' ' test_must_fail git -C test_implicit_1s2 fsmonitor--daemon status ' -# Confirm that MacOS hides all of the Unicode normalization and/or -# case folding from the FS events. That is, are the pathnames in the -# FS events reported using the spelling on the disk or in the spelling -# used by the other process. -# -# Note that we assume that the filesystem is set to case insensitive. -# -# NEEDSWORK: APFS handles Unicode and Unicode normalization -# differently than HFS+. I only have an APFS partition, so -# more testing here would be helpful. -# - -# Rename .git using alternate spelling and confirm that the daemon -# sees the event using the correct spelling and shutdown. -test_expect_success UTF8_NFD_TO_NFC 'MacOS event spelling (rename .GIT)' ' - test_when_finished "stop_daemon_delete_repo test_apfs" && - - git init test_apfs && - start_daemon test_apfs && - - test_path_is_dir test_apfs/.git && - test_path_is_dir test_apfs/.GIT && - - mv test_apfs/.GIT test_apfs/.FOO && - sleep 1 && - mv test_apfs/.FOO test_apfs/.git && - - test_must_fail git -C test_apfs fsmonitor--daemon status -' - test_expect_success 'cannot start multiple daemons' ' test_when_finished "stop_daemon_delete_repo test_multiple" && From 0e325759c69eaee54767cb70de797030e644998f Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:14:34 -0400 Subject: [PATCH 014/110] fixup! fsmonitor: handle shortname for .git This reverts commit 0fadc91ae493abac6950588d8ec1be2f2d905497. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-listen-win32.c | 192 ++++++---------------------- t/t7527-builtin-fsmonitor.sh | 65 ---------- 2 files changed, 40 insertions(+), 217 deletions(-) diff --git a/compat/fsmonitor/fsm-listen-win32.c b/compat/fsmonitor/fsm-listen-win32.c index 2431492c3a1c07..5cd4d136e64441 100644 --- a/compat/fsmonitor/fsm-listen-win32.c +++ b/compat/fsmonitor/fsm-listen-win32.c @@ -48,8 +48,6 @@ struct fsmonitor_daemon_backend_data #define LISTENER_HAVE_DATA_WORKTREE 1 #define LISTENER_HAVE_DATA_GITDIR 2 int nr_listener_handles; - - struct strbuf dot_git_shortname; }; /* @@ -260,62 +258,6 @@ static void cancel_rdcw_watch(struct one_watch *watch) watch->is_active = FALSE; } -/* - * Process a single relative pathname event. - * Return 1 if we should shutdown. - */ -static int process_1_worktree_event( - FILE_NOTIFY_INFORMATION *info, - struct string_list *cookie_list, - struct fsmonitor_batch **batch, - const struct strbuf *path, - enum fsmonitor_path_type t) -{ - const char *slash; - - switch (t) { - case IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX: - /* special case cookie files within .git */ - - /* Use just the filename of the cookie file. */ - slash = find_last_dir_sep(path->buf); - string_list_append(cookie_list, - slash ? slash + 1 : path->buf); - break; - - case IS_INSIDE_DOT_GIT: - /* ignore everything inside of "/.git/" */ - break; - - case IS_DOT_GIT: - /* "/.git" was deleted (or renamed away) */ - if ((info->Action == FILE_ACTION_REMOVED) || - (info->Action == FILE_ACTION_RENAMED_OLD_NAME)) { - trace2_data_string("fsmonitor", NULL, - "fsm-listen/dotgit", - "removed"); - return 1; - } - break; - - case IS_WORKDIR_PATH: - /* queue normal pathname */ - if (!*batch) - *batch = fsmonitor_batch__new(); - fsmonitor_batch__add_path(*batch, path->buf); - break; - - case IS_GITDIR: - case IS_INSIDE_GITDIR: - case IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX: - default: - BUG("unexpected path classification '%d' for '%s'", - t, path->buf); - } - - return 0; -} - /* * Process filesystem events that happen anywhere (recursively) under the * root directory. For a normal working directory, this includes @@ -360,6 +302,7 @@ static int process_worktree_events(struct fsmonitor_daemon_state *state) */ for (;;) { FILE_NOTIFY_INFORMATION *info = (void *)p; + const char *slash; enum fsmonitor_path_type t; strbuf_reset(&path); @@ -368,45 +311,45 @@ static int process_worktree_events(struct fsmonitor_daemon_state *state) t = fsmonitor_classify_path_workdir_relative(path.buf); - if (process_1_worktree_event(info, &cookie_list, &batch, - &path, t)) - goto force_shutdown; - - /* - * NEEDSWORK: If `path` contains a shortname (that is, - * if any component within it is a shortname), we - * should expand it to a longname (See - * `GetLongPathNameW()`) and re-normalize, classify, - * and process it because our client is probably - * expecting "normal" paths. - * - * HOWEVER, if our process has called `chdir()` to get - * us out of the root of the worktree (so that the - * root directory is not busy), then we have to be - * careful to convert the paths in the INFO array - * (which are relative to the directory of the RDCW - * watch and not the CWD) into absolute paths before - * calling GetLongPathNameW() and then convert the - * computed value back to a RDCW-relative pathname - * (which is what we and the client expect). - * - * FOR NOW, just handle case (1) exactly so that we - * shutdown properly when ".git" is deleted via the - * shortname alias. - * - * We might see case (2) events for cookie files, but - * we can ignore them. - * - * FOR LATER, handle case (3) where the worktree - * events contain shortnames. We should convert - * them to longnames to avoid confusing the client. - */ - if (data->dot_git_shortname.len && - !strcmp(path.buf, data->dot_git_shortname.buf) && - process_1_worktree_event(info, &cookie_list, &batch, - &data->dot_git_shortname, - IS_DOT_GIT)) - goto force_shutdown; + switch (t) { + case IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX: + /* special case cookie files within .git */ + + /* Use just the filename of the cookie file. */ + slash = find_last_dir_sep(path.buf); + string_list_append(&cookie_list, + slash ? slash + 1 : path.buf); + break; + + case IS_INSIDE_DOT_GIT: + /* ignore everything inside of "/.git/" */ + break; + + case IS_DOT_GIT: + /* "/.git" was deleted (or renamed away) */ + if ((info->Action == FILE_ACTION_REMOVED) || + (info->Action == FILE_ACTION_RENAMED_OLD_NAME)) { + trace2_data_string("fsmonitor", NULL, + "fsm-listen/dotgit", + "removed"); + goto force_shutdown; + } + break; + + case IS_WORKDIR_PATH: + /* queue normal pathname */ + if (!batch) + batch = fsmonitor_batch__new(); + fsmonitor_batch__add_path(batch, path.buf); + break; + + case IS_GITDIR: + case IS_INSIDE_GITDIR: + case IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX: + default: + BUG("unexpected path classification '%d' for '%s'", + t, path.buf); + } skip_this_path: if (!info->NextEntryOffset) @@ -480,14 +423,6 @@ static int process_gitdir_events(struct fsmonitor_daemon_state *state) t, path.buf); } - /* - * WRT shortnames, this external gitdir will not see - * case (1) nor case (3) events. - * - * We might see case (2) events for cookie files, but - * we can ignore them. - */ - skip_this_path: if (!info->NextEntryOffset) break; @@ -589,7 +524,6 @@ void fsm_listen__loop(struct fsmonitor_daemon_state *state) int fsm_listen__ctor(struct fsmonitor_daemon_state *state) { struct fsmonitor_daemon_backend_data *data; - char shortname[16]; /* a padded 8.3 buffer */ CALLOC_ARRAY(data, 1); @@ -620,52 +554,6 @@ int fsm_listen__ctor(struct fsmonitor_daemon_state *state) data->nr_listener_handles++; } - /* - * NEEDSWORK: Properly handle 8.3 shortnames. RDCW events can - * contain a shortname (if another application uses a - * shortname in a system call). We care about aliasing and - * the use of shortnames for: - * - * (1) ".git", - * -- if an external process deletes ".git" using "GIT~1", - * we need to catch that and shutdown. - * - * (2) our cookie files, - * -- if an external process deletes one of our cookie - * files using a shortname, we will get a shortname - * event for it. However, we should have already - * gotten a longname event for it when we created the - * cookie, so we can safely discard the shortname - * events for cookie files. - * - * (3) the spelling of modified files that we report to clients. - * -- we need to report the longname to the client because - * that is what they are expecting. Presumably, the - * client is going to lookup the paths that we report - * in their index and untracked-cache, so we should - * normalize the data for them. (Technically, they - * could adapt, so we could relax this maybe.) - * - * FOR NOW, while our CWD is at the root of the worktree we - * can easily get the spelling of the shortname of ".git" (if - * the volume has shortnames enabled). For most worktrees - * this value will be "GIT~1", but we don't want to assume - * that. - * - * Capture this so that we can handle (1). - * - * We leave (3) for a future effort. - */ - strbuf_init(&data->dot_git_shortname, 0); - GetShortPathNameA(".git", shortname, sizeof(shortname)); - if (!strcmp(".git", shortname)) - trace_printf_key(&trace_fsmonitor, "No shortname for '.git'"); - else { - trace_printf_key(&trace_fsmonitor, - "Shortname of '.git' is '%s'", shortname); - strbuf_addstr(&data->dot_git_shortname, shortname); - } - state->backend_data = data; return 0; diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index ee94861fe75d77..a43fc5190f5abe 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -123,71 +123,6 @@ test_expect_success 'implicit daemon stop (rename .git)' ' test_must_fail git -C test_implicit_2 fsmonitor--daemon status ' -# File systems on Windows may or may not have shortnames. -# This is a volume-specific setting on modern systems. -# "C:/" drives are required to have them enabled. Other -# hard drives default to disabled. -# -# This is a crude test to see if shortnames are enabled -# on the volume containing the test directory. It is -# crude, but it does not require elevation like `fsutil`. -# -test_lazy_prereq SHORTNAMES ' - mkdir .foo && - test -d "FOO~1" -' - -# Here we assume that the shortname of ".git" is "GIT~1". -test_expect_success MINGW,SHORTNAMES 'implicit daemon stop (rename GIT~1)' ' - test_when_finished "stop_daemon_delete_repo test_implicit_1s" && - - git init test_implicit_1s && - - start_daemon test_implicit_1s && - - # renaming the .git directory will implicitly stop the daemon. - # this moves {.git, GIT~1} to {.gitxyz, GITXYZ~1}. - # the rename-from FS Event will contain the shortname. - # - mv test_implicit_1s/GIT~1 test_implicit_1s/.gitxyz && - - sleep 1 && - # put it back so that our status will not crawl out to our - # parent directory. - # this moves {.gitxyz, GITXYZ~1} to {.git, GIT~1}. - mv test_implicit_1s/.gitxyz test_implicit_1s/.git && - - test_must_fail git -C test_implicit_1s fsmonitor--daemon status -' - -# Here we first create a file with LONGNAME of "GIT~1" before -# we create the repo. This will cause the shortname of ".git" -# to be "GIT~2". -test_expect_success MINGW,SHORTNAMES 'implicit daemon stop (rename GIT~2)' ' - test_when_finished "stop_daemon_delete_repo test_implicit_1s2" && - - mkdir test_implicit_1s2 && - echo HELLO >test_implicit_1s2/GIT~1 && - git init test_implicit_1s2 && - - test_path_is_file test_implicit_1s2/GIT~1 && - test_path_is_dir test_implicit_1s2/GIT~2 && - - start_daemon test_implicit_1s2 && - - # renaming the .git directory will implicitly stop the daemon. - # the rename-from FS Event will contain the shortname. - # - mv test_implicit_1s2/GIT~2 test_implicit_1s2/.gitxyz && - - sleep 1 && - # put it back so that our status will not crawl out to our - # parent directory. - mv test_implicit_1s2/.gitxyz test_implicit_1s2/.git && - - test_must_fail git -C test_implicit_1s2 fsmonitor--daemon status -' - test_expect_success 'cannot start multiple daemons' ' test_when_finished "stop_daemon_delete_repo test_multiple" && From ee717d5d946a8221d16ace974b0cfd9496897917 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:14:36 -0400 Subject: [PATCH 015/110] fixup! t7527: test status with untracked-cache and fsmonitor--daemon This reverts commit 27846ccfc583c9418bbab12896ca63da6c9f7372. Signed-off-by: Jeff Hostetler --- t/t7527-builtin-fsmonitor.sh | 87 ------------------------------------ 1 file changed, 87 deletions(-) diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index a43fc5190f5abe..eea9ca1a309ed3 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -162,8 +162,6 @@ test_expect_success 'setup' ' .gitignore expect* actual* - flush* - trace* EOF git -c core.useBuiltinFSMonitor= add . && @@ -510,89 +508,4 @@ test_expect_success 'cleanup worktrees' ' stop_daemon_delete_repo wt-base ' -# The next few tests perform arbitrary/contrived file operations and -# confirm that status is correct. That is, that the data (or lack of -# data) from fsmonitor doesn't cause incorrect results. And doesn't -# cause incorrect results when the untracked-cache is enabled. - -test_lazy_prereq UNTRACKED_CACHE ' - { git update-index --test-untracked-cache; ret=$?; } && - test $ret -ne 1 -' - -test_expect_success 'Matrix: setup for untracked-cache,fsmonitor matrix' ' - test_might_fail git config --unset core.useBuiltinFSMonitor && - git update-index --no-fsmonitor && - test_might_fail git fsmonitor--daemon stop -' - -matrix_clean_up_repo () { - git reset --hard HEAD - git clean -fd -} - -matrix_try () { - uc=$1 - fsm=$2 - fn=$3 - - test_expect_success "Matrix[uc:$uc][fsm:$fsm] $fn" ' - matrix_clean_up_repo && - $fn && - if test $uc = false -a $fsm = false - then - git status --porcelain=v1 >.git/expect.$fn - else - git status --porcelain=v1 >.git/actual.$fn - test_cmp .git/expect.$fn .git/actual.$fn - fi - ' - - return $? -} - -uc_values="false" -test_have_prereq UNTRACKED_CACHE && uc_values="false true" -for uc_val in $uc_values -do - if test $uc_val = false - then - test_expect_success "Matrix[uc:$uc_val] disable untracked cache" ' - git config core.untrackedcache false && - git update-index --no-untracked-cache - ' - else - test_expect_success "Matrix[uc:$uc_val] enable untracked cache" ' - git config core.untrackedcache true && - git update-index --untracked-cache - ' - fi - - fsm_values="false true" - for fsm_val in $fsm_values - do - if test $fsm_val = false - then - test_expect_success "Matrix[uc:$uc_val][fsm:$fsm_val] disable fsmonitor" ' - test_might_fail git config --unset core.useBuiltinFSMonitor && - git update-index --no-fsmonitor && - test_might_fail git fsmonitor--daemon stop 2>/dev/null - ' - else - test_expect_success "Matrix[uc:$uc_val][fsm:$fsm_val] enable fsmonitor" ' - git config core.useBuiltinFSMonitor true && - git fsmonitor--daemon start && - git update-index --fsmonitor - ' - fi - - matrix_try $uc_val $fsm_val edit_files - matrix_try $uc_val $fsm_val delete_files - matrix_try $uc_val $fsm_val create_files - matrix_try $uc_val $fsm_val rename_files - matrix_try $uc_val $fsm_val file_to_directory - matrix_try $uc_val $fsm_val directory_to_file - done -done - test_done From 4f40cd4a4aa51d3401bffd24b40c389e74a5b857 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:14:39 -0400 Subject: [PATCH 016/110] fixup! fsmonitor: force update index after large responses This reverts commit ec3dfa64cc1ec84275c5d85d427953eb4708364e. Signed-off-by: Jeff Hostetler --- fsmonitor.c | 50 +------------------------------------------------- 1 file changed, 1 insertion(+), 49 deletions(-) diff --git a/fsmonitor.c b/fsmonitor.c index 3a63cda58f73f0..695fb0ce4e7d75 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -233,45 +233,6 @@ static void fsmonitor_refresh_callback(struct index_state *istate, char *name) untracked_cache_invalidate_path(istate, name, 0); } -/* - * The number of pathnames that we need to receive from FSMonitor - * before we force the index to be updated. - * - * Note that any pathname within the set of received paths MAY cause - * cache-entry or istate flag bits to be updated and thus cause the - * index to be updated on disk. - * - * However, the response may contain many paths (such as ignored - * paths) that will not update any flag bits. And thus not force the - * index to be updated. (This is fine and normal.) It also means - * that the token will not be updated in the FSMonitor index - * extension. So the next Git command will find the same token in the - * index, make the same token-relative request, and receive the same - * response (plus any newly changed paths). If this response is large - * (and continues to grow), performance could be impacted. - * - * For example, if the user runs a build and it writes 100K object - * files but doesn't modify any source files, the index would not need - * to be updated. The FSMonitor response (after the build and - * relative to a pre-build token) might be 5MB. Each subsequent Git - * command will receive that same 100K/5MB response until something - * causes the index to be updated. And `refresh_fsmonitor()` will - * have to iterate over those 100K paths each time. - * - * Performance could be improved if we optionally force update the - * index after a very large response and get an updated token into - * the FSMonitor index extension. This should allow subsequent - * commands to get smaller and more current responses. - * - * The value chosen here does not need to be precise. The index - * will be updated automatically the first time the user touches - * a tracked file and causes a command like `git status` to - * update an mtime to be updated and/or set a flag bit. - * - * NEEDSWORK: Does this need to be a config value? - */ -static int fsmonitor_force_update_threshold = 100; - void refresh_fsmonitor(struct index_state *istate) { struct strbuf query_result = STRBUF_INIT; @@ -409,28 +370,19 @@ void refresh_fsmonitor(struct index_state *istate) * * This updates both the cache-entries and the untracked-cache. */ - int count = 0; - buf = query_result.buf; for (i = bol; i < query_result.len; i++) { if (buf[i] != '\0') continue; fsmonitor_refresh_callback(istate, buf + bol); bol = i + 1; - count++; } - if (bol < query_result.len) { + if (bol < query_result.len) fsmonitor_refresh_callback(istate, buf + bol); - count++; - } /* Now mark the untracked cache for fsmonitor usage */ if (istate->untracked) istate->untracked->use_fsmonitor = 1; - - if (count > fsmonitor_force_update_threshold) - istate->cache_changed |= FSMONITOR_CHANGED; - } else { /* * We received a trivial response, so invalidate everything. From 6cea160356c56dc1c058df2ec651962ba87d9b65 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:14:41 -0400 Subject: [PATCH 017/110] fixup! fsmonitor: enhance existing comments This reverts commit 76408441581b9fbc6f68ee509c568cc21c1653de. Signed-off-by: Jeff Hostetler --- fsmonitor.c | 37 +++++++------------------------------ 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/fsmonitor.c b/fsmonitor.c index 695fb0ce4e7d75..af3c134e7d7082 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -351,25 +351,9 @@ void refresh_fsmonitor(struct index_state *istate) } apply_results: - /* - * The response from FSMonitor (excluding the header token) is - * either: - * - * [a] a (possibly empty) list of NUL delimited relative - * pathnames of changed paths. This list can contain - * files and directories. Directories have a trailing - * slash. - * - * [b] a single '/' to indicate the provider had no - * information and that we should consider everything - * invalid. We call this a trivial response. - */ + /* a fsmonitor process can return '/' to indicate all entries are invalid */ if (query_success && query_result.buf[bol] != '/') { - /* - * Mark all pathnames returned by the monitor as dirty. - * - * This updates both the cache-entries and the untracked-cache. - */ + /* Mark all entries returned by the monitor as dirty */ buf = query_result.buf; for (i = bol; i < query_result.len; i++) { if (buf[i] != '\0') @@ -384,15 +368,11 @@ void refresh_fsmonitor(struct index_state *istate) if (istate->untracked) istate->untracked->use_fsmonitor = 1; } else { - /* - * We received a trivial response, so invalidate everything. - * - * We only want to run the post index changed hook if - * we've actually changed entries, so keep track if we - * actually changed entries or not. - */ - int is_cache_changed = 0; + /* We only want to run the post index changed hook if we've actually changed entries, so keep track + * if we actually changed entries or not */ + int is_cache_changed = 0; + /* Mark all entries invalid */ for (i = 0; i < istate->cache_nr; i++) { if (istate->cache[i]->ce_flags & CE_FSMONITOR_VALID) { is_cache_changed = 1; @@ -400,10 +380,7 @@ void refresh_fsmonitor(struct index_state *istate) } } - /* - * If we're going to check every file, ensure we save - * the results. - */ + /* If we're going to check every file, ensure we save the results */ if (is_cache_changed) istate->cache_changed |= FSMONITOR_CHANGED; From 39e04891763ad10f05fc40f73191829b74f7879d Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:14:44 -0400 Subject: [PATCH 018/110] fixup! fsmonitor--daemon: use a cookie file to sync with file system This reverts commit 0de22628c5926892dd2d80521331bae9acfe409a. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 228 +----------------------------------- fsmonitor--daemon.h | 5 - 2 files changed, 1 insertion(+), 232 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index a4e3f6e3099ff3..2e08f430d04456 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -94,149 +94,6 @@ static int do_as_client__status(void) } } -enum fsmonitor_cookie_item_result { - FCIR_ERROR = -1, /* could not create cookie file ? */ - FCIR_INIT = 0, - FCIR_SEEN, - FCIR_ABORT, -}; - -struct fsmonitor_cookie_item { - struct hashmap_entry entry; - const char *name; - enum fsmonitor_cookie_item_result result; -}; - -static int cookies_cmp(const void *data, const struct hashmap_entry *he1, - const struct hashmap_entry *he2, const void *keydata) -{ - const struct fsmonitor_cookie_item *a = - container_of(he1, const struct fsmonitor_cookie_item, entry); - const struct fsmonitor_cookie_item *b = - container_of(he2, const struct fsmonitor_cookie_item, entry); - - return strcmp(a->name, keydata ? keydata : b->name); -} - -static enum fsmonitor_cookie_item_result with_lock__wait_for_cookie( - struct fsmonitor_daemon_state *state) -{ - /* assert current thread holding state->main_lock */ - - int fd; - struct fsmonitor_cookie_item *cookie; - struct strbuf cookie_pathname = STRBUF_INIT; - struct strbuf cookie_filename = STRBUF_INIT; - enum fsmonitor_cookie_item_result result; - int my_cookie_seq; - - CALLOC_ARRAY(cookie, 1); - - my_cookie_seq = state->cookie_seq++; - - strbuf_addf(&cookie_filename, "%i-%i", getpid(), my_cookie_seq); - - strbuf_addbuf(&cookie_pathname, &state->path_cookie_prefix); - strbuf_addbuf(&cookie_pathname, &cookie_filename); - - cookie->name = strbuf_detach(&cookie_filename, NULL); - cookie->result = FCIR_INIT; - hashmap_entry_init(&cookie->entry, strhash(cookie->name)); - - hashmap_add(&state->cookies, &cookie->entry); - - trace_printf_key(&trace_fsmonitor, "cookie-wait: '%s' '%s'", - cookie->name, cookie_pathname.buf); - - /* - * Create the cookie file on disk and then wait for a notification - * that the listener thread has seen it. - */ - fd = open(cookie_pathname.buf, O_WRONLY | O_CREAT | O_EXCL, 0600); - if (fd >= 0) { - close(fd); - unlink(cookie_pathname.buf); - - /* - * NEEDSWORK: This is an infinite wait (well, unless another - * thread sends us an abort). I'd like to change this to - * use `pthread_cond_timedwait()` and return an error/timeout - * and let the caller do the trivial response thing. - */ - while (cookie->result == FCIR_INIT) - pthread_cond_wait(&state->cookies_cond, - &state->main_lock); - } else { - error_errno(_("could not create fsmonitor cookie '%s'"), - cookie->name); - - cookie->result = FCIR_ERROR; - } - - hashmap_remove(&state->cookies, &cookie->entry, NULL); - - result = cookie->result; - - free((char*)cookie->name); - free(cookie); - strbuf_release(&cookie_pathname); - - return result; -} - -/* - * Mark these cookies as _SEEN and wake up the corresponding client threads. - */ -static void with_lock__mark_cookies_seen(struct fsmonitor_daemon_state *state, - const struct string_list *cookie_names) -{ - /* assert current thread holding state->main_lock */ - - int k; - int nr_seen = 0; - - for (k = 0; k < cookie_names->nr; k++) { - struct fsmonitor_cookie_item key; - struct fsmonitor_cookie_item *cookie; - - key.name = cookie_names->items[k].string; - hashmap_entry_init(&key.entry, strhash(key.name)); - - cookie = hashmap_get_entry(&state->cookies, &key, entry, NULL); - if (cookie) { - trace_printf_key(&trace_fsmonitor, "cookie-seen: '%s'", - cookie->name); - cookie->result = FCIR_SEEN; - nr_seen++; - } - } - - if (nr_seen) - pthread_cond_broadcast(&state->cookies_cond); -} - -/* - * Set _ABORT on all pending cookies and wake up all client threads. - */ -static void with_lock__abort_all_cookies(struct fsmonitor_daemon_state *state) -{ - /* assert current thread holding state->main_lock */ - - struct hashmap_iter iter; - struct fsmonitor_cookie_item *cookie; - int nr_aborted = 0; - - hashmap_for_each_entry(&state->cookies, &iter, cookie, entry) { - trace_printf_key(&trace_fsmonitor, "cookie-abort: '%s'", - cookie->name); - cookie->result = FCIR_ABORT; - nr_aborted++; - } - - if (nr_aborted) - pthread_cond_broadcast(&state->cookies_cond); -} - /* * Requests to and from a FSMonitor Protocol V2 provider use an opaque * "token" as a virtual timestamp. Clients can request a summary of all @@ -552,9 +409,6 @@ static void fsmonitor_free_token_data(struct fsmonitor_token_data *token) * We should create a new token and start fresh (as if we just * booted up). * - * [2] Some of those lost events may have been for cookie files. We - * should assume the worst and abort them rather letting them starve. - * * If there are no concurrent threads readering the current token data * series, we can free it now. Otherwise, let the last reader free * it. @@ -576,8 +430,6 @@ static void with_lock__do_force_resync(struct fsmonitor_daemon_state *state) state->current_token_data = new_one; fsmonitor_free_token_data(free_me); - - with_lock__abort_all_cookies(state); } void fsmonitor_force_resync(struct fsmonitor_daemon_state *state) @@ -653,8 +505,6 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, int hash_ret; int do_trivial = 0; int do_flush = 0; - int do_cookie = 0; - enum fsmonitor_cookie_item_result cookie_result; /* * We expect `command` to be of the form: @@ -715,7 +565,6 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, * We have a V2 valid token: * "builtin::" */ - do_cookie = 1; } } @@ -724,30 +573,6 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, if (!state->current_token_data) BUG("fsmonitor state does not have a current token"); - /* - * Write a cookie file inside the directory being watched in - * an effort to flush out existing filesystem events that we - * actually care about. Suspend this client thread until we - * see the filesystem events for this cookie file. - * - * Creating the cookie lets us guarantee that our FS listener - * thread has drained the kernel queue and we are caught up - * with the kernel. - * - * If we cannot create the cookie (or otherwise guarantee that - * we are caught up), we send a trivial response. We have to - * assume that there might be some very, very recent activity - * on the FS still in flight. - */ - if (do_cookie) { - cookie_result = with_lock__wait_for_cookie(state); - if (cookie_result != FCIR_SEEN) { - error(_("fsmonitor: cookie_result '%d' != SEEN"), - cookie_result); - do_trivial = 1; - } - } - if (do_flush) with_lock__do_force_resync(state); @@ -961,9 +786,7 @@ static int handle_client(void *data, return result; } -#define FSMONITOR_DIR "fsmonitor--daemon" -#define FSMONITOR_COOKIE_DIR "cookies" -#define FSMONITOR_COOKIE_PREFIX (FSMONITOR_DIR "/" FSMONITOR_COOKIE_DIR "/") +#define FSMONITOR_COOKIE_PREFIX ".fsmonitor-daemon-" enum fsmonitor_path_type fsmonitor_classify_path_workdir_relative( const char *rel) @@ -1116,9 +939,6 @@ void fsmonitor_publish(struct fsmonitor_daemon_state *state, } } - if (cookie_names->nr) - with_lock__mark_cookies_seen(state, cookie_names); - pthread_mutex_unlock(&state->main_lock); } @@ -1208,9 +1028,7 @@ static int fsmonitor_run_daemon(void) memset(&state, 0, sizeof(state)); - hashmap_init(&state.cookies, cookies_cmp, NULL, 0); pthread_mutex_init(&state.main_lock, NULL); - pthread_cond_init(&state.cookies_cond, NULL); state.error_code = 0; state.current_token_data = fsmonitor_new_token_data(); @@ -1235,44 +1053,6 @@ static int fsmonitor_run_daemon(void) state.nr_paths_watching = 2; } - /* - * We will write filesystem syncing cookie files into - * ///-. - * - * The extra layers of subdirectories here keep us from - * changing the mtime on ".git/" or ".git/foo/" when we create - * or delete cookie files. - * - * There have been problems with some IDEs that do a - * non-recursive watch of the ".git/" directory and run a - * series of commands any time something happens. - * - * For example, if we place our cookie files directly in - * ".git/" or ".git/foo/" then a `git status` (or similar - * command) from the IDE will cause a cookie file to be - * created in one of those dirs. This causes the mtime of - * those dirs to change. This triggers the IDE's watch - * notification. This triggers the IDE to run those commands - * again. And the process repeats and the machine never goes - * idle. - * - * Adding the extra layers of subdirectories prevents the - * mtime of ".git/" and ".git/foo" from changing when a - * cookie file is created. - */ - strbuf_init(&state.path_cookie_prefix, 0); - strbuf_addbuf(&state.path_cookie_prefix, &state.path_gitdir_watch); - - strbuf_addch(&state.path_cookie_prefix, '/'); - strbuf_addstr(&state.path_cookie_prefix, FSMONITOR_DIR); - mkdir(state.path_cookie_prefix.buf, 0777); - - strbuf_addch(&state.path_cookie_prefix, '/'); - strbuf_addstr(&state.path_cookie_prefix, FSMONITOR_COOKIE_DIR); - mkdir(state.path_cookie_prefix.buf, 0777); - - strbuf_addch(&state.path_cookie_prefix, '/'); - /* * Confirm that we can create platform-specific resources for the * filesystem listener before we bother starting all the threads. @@ -1285,7 +1065,6 @@ static int fsmonitor_run_daemon(void) err = fsmonitor_run_daemon_1(&state); done: - pthread_cond_destroy(&state.cookies_cond); pthread_mutex_destroy(&state.main_lock); fsm_listen__dtor(&state); @@ -1293,11 +1072,6 @@ static int fsmonitor_run_daemon(void) strbuf_release(&state.path_worktree_watch); strbuf_release(&state.path_gitdir_watch); - strbuf_release(&state.path_cookie_prefix); - - /* - * NEEDSWORK: Consider "rm -rf /" - */ return err; } diff --git a/fsmonitor--daemon.h b/fsmonitor--daemon.h index e9fc099bae9cfb..89a9ef20b24b45 100644 --- a/fsmonitor--daemon.h +++ b/fsmonitor--daemon.h @@ -45,11 +45,6 @@ struct fsmonitor_daemon_state { struct fsmonitor_token_data *current_token_data; - struct strbuf path_cookie_prefix; - pthread_cond_t cookies_cond; - int cookie_seq; - struct hashmap cookies; - int error_code; struct fsmonitor_daemon_backend_data *backend_data; From 0dae22b88f2f517056d0620c544242e0597ca54e Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:14:47 -0400 Subject: [PATCH 019/110] fixup! fsmonitor--daemon: periodically truncate list of modified files This reverts commit 20eea488e8f14ca5756534a8406d377bd74233b9. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 95 ------------------------------------- 1 file changed, 95 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 2e08f430d04456..d85157f56c0f4d 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -300,89 +300,6 @@ static void fsmonitor_batch__combine(struct fsmonitor_batch *batch_dest, batch_src->interned_paths[k]; } -/* - * To keep the batch list from growing unbounded in response to filesystem - * activity, we try to truncate old batches from the end of the list as - * they become irrelevant. - * - * We assume that the .git/index will be updated with the most recent token - * any time the index is updated. And future commands will only ask for - * recent changes *since* that new token. So as tokens advance into the - * future, older batch items will never be requested/needed. So we can - * truncate them without loss of functionality. - * - * However, multiple commands may be talking to the daemon concurrently - * or perform a slow command, so a little "token skew" is possible. - * Therefore, we want this to be a little bit lazy and have a generous - * delay. - * - * The current reader thread walked backwards in time from `token->batch_head` - * back to `batch_marker` somewhere in the middle of the batch list. - * - * Let's walk backwards in time from that marker an arbitrary delay - * and truncate the list there. Note that these timestamps are completely - * artificial (based on when we pinned the batch item) and not on any - * filesystem activity. - * - * Return the obsolete portion of the list after we have removed it from - * the official list so that the caller can free it after leaving the lock. - */ -#define MY_TIME_DELAY_SECONDS (5 * 60) /* seconds */ - -static struct fsmonitor_batch *with_lock__truncate_old_batches( - struct fsmonitor_daemon_state *state, - const struct fsmonitor_batch *batch_marker) -{ - /* assert current thread holding state->main_lock */ - - const struct fsmonitor_batch *batch; - struct fsmonitor_batch *remainder; - - if (!batch_marker) - return NULL; - - trace_printf_key(&trace_fsmonitor, "Truncate: mark (%"PRIu64",%"PRIu64")", - batch_marker->batch_seq_nr, - (uint64_t)batch_marker->pinned_time); - - for (batch = batch_marker; batch; batch = batch->next) { - time_t t; - - if (!batch->pinned_time) /* an overflow batch */ - continue; - - t = batch->pinned_time + MY_TIME_DELAY_SECONDS; - if (t > batch_marker->pinned_time) /* too close to marker */ - continue; - - goto truncate_past_here; - } - - return NULL; - -truncate_past_here: - state->current_token_data->batch_tail = (struct fsmonitor_batch *)batch; - - remainder = ((struct fsmonitor_batch *)batch)->next; - ((struct fsmonitor_batch *)batch)->next = NULL; - - return remainder; -} - -static void free_remainder(struct fsmonitor_batch *remainder) -{ - struct fsmonitor_batch *p; - - if (!remainder) - return; - - for (p = remainder; p; p = fsmonitor_batch__pop(p)) { - trace_printf_key(&trace_fsmonitor, - "Truncate: kill (%"PRIu64",%"PRIu64")", - p->batch_seq_nr, (uint64_t)p->pinned_time); - } -} - static void fsmonitor_free_token_data(struct fsmonitor_token_data *token) { struct fsmonitor_batch *p; @@ -499,7 +416,6 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, const char *p; const struct fsmonitor_batch *batch_head; const struct fsmonitor_batch *batch; - struct fsmonitor_batch *remainder = NULL; intmax_t count = 0, duplicates = 0; kh_str_t *shown; int hash_ret; @@ -729,22 +645,11 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, * that work. */ fsmonitor_free_token_data(token_data); - } else if (batch) { - /* - * This batch is the first item in the list - * that is older than the requested sequence - * number and might be considered to be - * obsolete. See if we can truncate the list - * and save some memory. - */ - remainder = with_lock__truncate_old_batches(state, batch); } } pthread_mutex_unlock(&state->main_lock); - free_remainder(remainder); - trace2_data_intmax("fsmonitor", the_repository, "response/length", total_response_len); trace2_data_intmax("fsmonitor", the_repository, "response/count/files", count); trace2_data_intmax("fsmonitor", the_repository, "response/count/duplicates", duplicates); From bfc1efc0d6001528325f738e9285064273a1e0b1 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:14:49 -0400 Subject: [PATCH 020/110] fixup! t7527: create test for fsmonitor--daemon This reverts commit 8318d7dec0a1a1eabc8897573e4b5d32fd428fd9. Signed-off-by: Jeff Hostetler --- t/t7527-builtin-fsmonitor.sh | 511 ----------------------------------- 1 file changed, 511 deletions(-) delete mode 100755 t/t7527-builtin-fsmonitor.sh diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh deleted file mode 100755 index eea9ca1a309ed3..00000000000000 --- a/t/t7527-builtin-fsmonitor.sh +++ /dev/null @@ -1,511 +0,0 @@ -#!/bin/sh - -test_description='built-in file system watcher' - -. ./test-lib.sh - -if ! test_have_prereq FSMONITOR_DAEMON -then - skip_all="fsmonitor--daemon is not supported on this platform" - test_done -fi - -stop_daemon_delete_repo () { - r=$1 - git -C $r fsmonitor--daemon stop >/dev/null 2>/dev/null - rm -rf $1 - return 0 -} - -start_daemon () { - case "$#" in - 1) r="-C $1";; - *) r=""; - esac - - git $r fsmonitor--daemon start || return $? - git $r fsmonitor--daemon status || return $? - - return 0 -} - -# Is a Trace2 data event present with the given catetory and key? -# We do not care what the value is. -# -have_t2_data_event () { - c=$1 - k=$2 - - grep -e '"event":"data".*"category":"'"$c"'".*"key":"'"$k"'"' -} - -test_expect_success 'explicit daemon start and stop' ' - test_when_finished "stop_daemon_delete_repo test_explicit" && - - git init test_explicit && - start_daemon test_explicit && - - git -C test_explicit fsmonitor--daemon stop && - test_must_fail git -C test_explicit fsmonitor--daemon status -' - -test_expect_success 'implicit daemon start' ' - test_when_finished "stop_daemon_delete_repo test_implicit" && - - git init test_implicit && - test_must_fail git -C test_implicit fsmonitor--daemon status && - - # query will implicitly start the daemon. - # - # for test-script simplicity, we send a V1 timestamp rather than - # a V2 token. either way, the daemon response to any query contains - # a new V2 token. (the daemon may complain that we sent a V1 request, - # but this test case is only concerned with whether the daemon was - # implicitly started.) - - GIT_TRACE2_EVENT="$(pwd)/.git/trace" \ - test-tool -C test_implicit fsmonitor-client query --token 0 >actual && - nul_to_q actual.filtered && - grep "builtin:" actual.filtered && - - # confirm that a daemon was started in the background. - # - # since the mechanism for starting the background daemon is platform - # dependent, just confirm that the foreground command received a - # response from the daemon. - - have_t2_data_event fsm_client query/response-length <.git/trace && - - git -C test_implicit fsmonitor--daemon status && - git -C test_implicit fsmonitor--daemon stop && - test_must_fail git -C test_implicit fsmonitor--daemon status -' - -test_expect_success 'implicit daemon stop (delete .git)' ' - test_when_finished "stop_daemon_delete_repo test_implicit_1" && - - git init test_implicit_1 && - - start_daemon test_implicit_1 && - - # deleting the .git directory will implicitly stop the daemon. - rm -rf test_implicit_1/.git && - - # [1] Create an empty .git directory so that the following Git - # command will stay relative to the `-C` directory. - # - # Without this, the Git command will override the requested - # -C argument and crawl out to the containing Git source tree. - # This would make the test result dependent upon whether we - # were using fsmonitor on our development worktree. - # - sleep 1 && - mkdir test_implicit_1/.git && - - test_must_fail git -C test_implicit_1 fsmonitor--daemon status -' - -test_expect_success 'implicit daemon stop (rename .git)' ' - test_when_finished "stop_daemon_delete_repo test_implicit_2" && - - git init test_implicit_2 && - - start_daemon test_implicit_2 && - - # renaming the .git directory will implicitly stop the daemon. - mv test_implicit_2/.git test_implicit_2/.xxx && - - # See [1] above. - # - sleep 1 && - mkdir test_implicit_2/.git && - - test_must_fail git -C test_implicit_2 fsmonitor--daemon status -' - -test_expect_success 'cannot start multiple daemons' ' - test_when_finished "stop_daemon_delete_repo test_multiple" && - - git init test_multiple && - - start_daemon test_multiple && - - test_must_fail git -C test_multiple fsmonitor--daemon start 2>actual && - grep "fsmonitor--daemon is already running" actual && - - git -C test_multiple fsmonitor--daemon stop && - test_must_fail git -C test_multiple fsmonitor--daemon status -' - -# These tests use the main repo in the trash directory - -test_expect_success 'setup' ' - >tracked && - >modified && - >delete && - >rename && - mkdir dir1 && - >dir1/tracked && - >dir1/modified && - >dir1/delete && - >dir1/rename && - mkdir dir2 && - >dir2/tracked && - >dir2/modified && - >dir2/delete && - >dir2/rename && - mkdir dirtorename && - >dirtorename/a && - >dirtorename/b && - - cat >.gitignore <<-\EOF && - .gitignore - expect* - actual* - EOF - - git -c core.useBuiltinFSMonitor= add . && - test_tick && - git -c core.useBuiltinFSMonitor= commit -m initial && - - git config core.useBuiltinFSMonitor true -' - -# The test already explicitly stopped (or tried to stop) the daemon. -# This is here in case something else fails first. -# -redundant_stop_daemon () { - git fsmonitor--daemon stop - return 0 -} - -test_expect_success 'update-index implicitly starts daemon' ' - test_when_finished redundant_stop_daemon && - - test_must_fail git fsmonitor--daemon status && - - GIT_TRACE2_EVENT="$(pwd)/.git/trace_implicit_1" \ - git update-index --fsmonitor && - - git fsmonitor--daemon status && - test_might_fail git fsmonitor--daemon stop && - - # Confirm that the trace2 log contains a record of the - # daemon starting. - test_subcommand git fsmonitor--daemon start <.git/trace_implicit_1 -' - -test_expect_success 'status implicitly starts daemon' ' - test_when_finished redundant_stop_daemon && - - test_must_fail git fsmonitor--daemon status && - - GIT_TRACE2_EVENT="$(pwd)/.git/trace_implicit_2" \ - git status >actual && - - git fsmonitor--daemon status && - test_might_fail git fsmonitor--daemon stop && - - # Confirm that the trace2 log contains a record of the - # daemon starting. - test_subcommand git fsmonitor--daemon start <.git/trace_implicit_2 -' - -edit_files() { - echo 1 >modified - echo 2 >dir1/modified - echo 3 >dir2/modified - >dir1/untracked -} - -delete_files() { - rm -f delete - rm -f dir1/delete - rm -f dir2/delete -} - -create_files() { - echo 1 >new - echo 2 >dir1/new - echo 3 >dir2/new -} - -rename_files() { - mv rename renamed - mv dir1/rename dir1/renamed - mv dir2/rename dir2/renamed -} - -file_to_directory() { - rm -f delete - mkdir delete - echo 1 >delete/new -} - -directory_to_file() { - rm -rf dir1 - echo 1 >dir1 -} - -verify_status() { - git status >actual && - GIT_INDEX_FILE=.git/fresh-index git read-tree master && - GIT_INDEX_FILE=.git/fresh-index git -c core.useBuiltinFSMonitor= status >expect && - test_cmp expect actual && - echo HELLO AFTER && - cat .git/trace && - echo HELLO AFTER -} - -# The next few test cases confirm that our fsmonitor daemon sees each type -# of OS filesystem notification that we care about. At this layer we just -# ensure we are getting the OS notifications and do not try to confirm what -# is reported by `git status`. -# -# We run a simple query after modifying the filesystem just to introduce -# a bit of a delay so that the trace logging from the daemon has time to -# get flushed to disk. -# -# We `reset` and `clean` at the bottom of each test (and before stopping the -# daemon) because these commands might implicitly restart the daemon. - -clean_up_repo_and_stop_daemon () { - git reset --hard HEAD - git clean -fd - git fsmonitor--daemon stop - rm -f .git/trace -} - -test_expect_success 'edit some files' ' - test_when_finished clean_up_repo_and_stop_daemon && - - ( - GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && - export GIT_TRACE_FSMONITOR && - - start_daemon - ) && - - edit_files && - - test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && - - grep "^event: dir1/modified$" .git/trace && - grep "^event: dir2/modified$" .git/trace && - grep "^event: modified$" .git/trace && - grep "^event: dir1/untracked$" .git/trace -' - -test_expect_success 'create some files' ' - test_when_finished clean_up_repo_and_stop_daemon && - - ( - GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && - export GIT_TRACE_FSMONITOR && - - start_daemon - ) && - - create_files && - - test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && - - grep "^event: dir1/new$" .git/trace && - grep "^event: dir2/new$" .git/trace && - grep "^event: new$" .git/trace -' - -test_expect_success 'delete some files' ' - test_when_finished clean_up_repo_and_stop_daemon && - - ( - GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && - export GIT_TRACE_FSMONITOR && - - start_daemon - ) && - - delete_files && - - test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && - - grep "^event: dir1/delete$" .git/trace && - grep "^event: dir2/delete$" .git/trace && - grep "^event: delete$" .git/trace -' - -test_expect_success 'rename some files' ' - test_when_finished clean_up_repo_and_stop_daemon && - - ( - GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && - export GIT_TRACE_FSMONITOR && - - start_daemon - ) && - - rename_files && - - test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && - - grep "^event: dir1/rename$" .git/trace && - grep "^event: dir2/rename$" .git/trace && - grep "^event: rename$" .git/trace && - grep "^event: dir1/renamed$" .git/trace && - grep "^event: dir2/renamed$" .git/trace && - grep "^event: renamed$" .git/trace -' - -test_expect_success 'rename directory' ' - test_when_finished clean_up_repo_and_stop_daemon && - - ( - GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && - export GIT_TRACE_FSMONITOR && - - start_daemon - ) && - - mv dirtorename dirrenamed && - - test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && - - grep "^event: dirtorename/*$" .git/trace && - grep "^event: dirrenamed/*$" .git/trace -' - -test_expect_success 'file changes to directory' ' - test_when_finished clean_up_repo_and_stop_daemon && - - ( - GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && - export GIT_TRACE_FSMONITOR && - - start_daemon - ) && - - file_to_directory && - - test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && - - grep "^event: delete$" .git/trace && - grep "^event: delete/new$" .git/trace -' - -test_expect_success 'directory changes to a file' ' - test_when_finished clean_up_repo_and_stop_daemon && - - ( - GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && - export GIT_TRACE_FSMONITOR && - - start_daemon - ) && - - directory_to_file && - - test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && - - grep "^event: dir1$" .git/trace -' - -# The next few test cases exercise the token-resync code. When filesystem -# drops events (because of filesystem velocity or because the daemon isn't -# polling fast enough), we need to discard the cached data (relative to the -# current token) and start collecting events under a new token. -# -# the 'test-tool fsmonitor-client flush' command can be used to send a -# "flush" message to a running daemon and ask it to do a flush/resync. - -test_expect_success 'flush cached data' ' - test_when_finished "stop_daemon_delete_repo test_flush" && - - git init test_flush && - - ( - GIT_TEST_FSMONITOR_TOKEN=true && - export GIT_TEST_FSMONITOR_TOKEN && - - GIT_TRACE_FSMONITOR="$(pwd)/.git/trace_daemon" && - export GIT_TRACE_FSMONITOR && - - start_daemon test_flush - ) && - - # The daemon should have an initial token with no events in _0 and - # then a few (probably platform-specific number of) events in _1. - # These should both have the same . - - test-tool -C test_flush fsmonitor-client query --token "builtin:test_00000001:0" >actual_0 && - nul_to_q actual_q0 && - - touch test_flush/file_1 && - touch test_flush/file_2 && - - test-tool -C test_flush fsmonitor-client query --token "builtin:test_00000001:0" >actual_1 && - nul_to_q actual_q1 && - - grep "file_1" actual_q1 && - - # Force a flush. This will change the , reset the , and - # flush the file data. Then create some events and ensure that the file - # again appears in the cache. It should have the new . - - test-tool -C test_flush fsmonitor-client flush >flush_0 && - nul_to_q flush_q0 && - grep "^builtin:test_00000002:0Q/Q$" flush_q0 && - - test-tool -C test_flush fsmonitor-client query --token "builtin:test_00000002:0" >actual_2 && - nul_to_q actual_q2 && - - grep "^builtin:test_00000002:0Q$" actual_q2 && - - touch test_flush/file_3 && - - test-tool -C test_flush fsmonitor-client query --token "builtin:test_00000002:0" >actual_3 && - nul_to_q actual_q3 && - - grep "file_3" actual_q3 -' - -# The next few test cases create repos where the .git directory is NOT -# inside the one of the working directory. That is, where .git is a file -# that points to a directory elsewhere. This happens for submodules and -# non-primary worktrees. - -test_expect_success 'setup worktree base' ' - git init wt-base && - echo 1 >wt-base/file1 && - git -C wt-base add file1 && - git -C wt-base commit -m "c1" -' - -test_expect_success 'worktree with .git file' ' - git -C wt-base worktree add ../wt-secondary && - - ( - GIT_TRACE2_PERF="$(pwd)/trace2_wt_secondary" && - export GIT_TRACE2_PERF && - - GIT_TRACE_FSMONITOR="$(pwd)/trace_wt_secondary" && - export GIT_TRACE_FSMONITOR && - - start_daemon wt-secondary - ) && - - git -C wt-secondary fsmonitor--daemon stop && - test_must_fail git -C wt-secondary fsmonitor--daemon status -' - -# NEEDSWORK: Repeat one of the "edit" tests on wt-secondary and -# confirm that we get the same events and behavior -- that is, that -# fsmonitor--daemon correctly watches BOTH the working directory and -# the external GITDIR directory and behaves the same as when ".git" -# is a directory inside the working directory. - -test_expect_success 'cleanup worktrees' ' - stop_daemon_delete_repo wt-secondary && - stop_daemon_delete_repo wt-base -' - -test_done From b1488adc281ad97bbaee886215fbca9067700559 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:14:51 -0400 Subject: [PATCH 021/110] fixup! t/perf/p7519: add fsmonitor--daemon test cases This reverts commit 41b215e19afda5850f77d340638671375f90254c. Signed-off-by: Jeff Hostetler --- t/perf/p7519-fsmonitor.sh | 37 +++---------------------------------- 1 file changed, 3 insertions(+), 34 deletions(-) diff --git a/t/perf/p7519-fsmonitor.sh b/t/perf/p7519-fsmonitor.sh index e70252ed65ab70..171644ffc90761 100755 --- a/t/perf/p7519-fsmonitor.sh +++ b/t/perf/p7519-fsmonitor.sh @@ -24,8 +24,7 @@ test_description="Test core.fsmonitor" # GIT_PERF_7519_SPLIT_INDEX: used to configure core.splitIndex # GIT_PERF_7519_FSMONITOR: used to configure core.fsMonitor. May be an # absolute path to an integration. May be a space delimited list of -# absolute paths to integrations. (This hook or list of hooks does not -# include the built-in fsmonitor--daemon.) +# absolute paths to integrations. # # The big win for using fsmonitor is the elimination of the need to scan the # working directory looking for changed and untracked files. If the file @@ -144,16 +143,10 @@ test_expect_success "one time repo setup" ' setup_for_fsmonitor() { # set INTEGRATION_SCRIPT depending on the environment - if test -n "$USE_FSMONITOR_DAEMON" + if test -n "$INTEGRATION_PATH" then - git config core.useBuiltinFSMonitor true && - INTEGRATION_SCRIPT=false - elif test -n "$INTEGRATION_PATH" - then - git config core.useBuiltinFSMonitor false && INTEGRATION_SCRIPT="$INTEGRATION_PATH" else - git config core.useBuiltinFSMonitor false && # # Choose integration script based on existence of Watchman. # Fall back to an empty integration script. @@ -189,10 +182,7 @@ test_perf_w_drop_caches () { } test_fsmonitor_suite() { - if test -n "$USE_FSMONITOR_DAEMON" - then - DESC="builtin fsmonitor--daemon" - elif test -n "$INTEGRATION_SCRIPT"; then + if test -n "$INTEGRATION_SCRIPT"; then DESC="fsmonitor=$(basename $INTEGRATION_SCRIPT)" else DESC="fsmonitor=disabled" @@ -303,25 +293,4 @@ test_expect_success "setup without fsmonitor" ' test_fsmonitor_suite trace_stop -# -# Run a full set of perf tests using the built-in fsmonitor--daemon. -# It does not use the Hook API, so it has a different setup. -# Explicitly start the daemon here and before we start client commands -# so that we can later add custom tracing. -# -if test_have_prereq FSMONITOR_DAEMON -then - USE_FSMONITOR_DAEMON=t - - trace_start fsmonitor--daemon--server - git fsmonitor--daemon start - - trace_start fsmonitor--daemon--client - test_expect_success "setup for fsmonitor--daemon" 'setup_for_fsmonitor' - test_fsmonitor_suite - - git fsmonitor--daemon stop - trace_stop -fi - test_done From 62cad115d754b528dd8834eb40e221ac0fedd311 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:14:54 -0400 Subject: [PATCH 022/110] fixup! t/perf: avoid copying builtin fsmonitor files into test repo This reverts commit 5ab490f926b04e415b60bfdf4e1b989fcfdffe40. Signed-off-by: Jeff Hostetler --- t/perf/perf-lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/perf/perf-lib.sh b/t/perf/perf-lib.sh index dc29df12fd5fda..f5ed092ee591ca 100644 --- a/t/perf/perf-lib.sh +++ b/t/perf/perf-lib.sh @@ -74,7 +74,7 @@ test_perf_copy_repo_contents () { for stuff in "$1"/* do case "$stuff" in - */objects|*/hooks|*/config|*/commondir|*/gitdir|*/worktrees|*/fsmonitor--daemon*) + */objects|*/hooks|*/config|*/commondir|*/gitdir|*/worktrees) ;; *) cp -R "$stuff" "$repo/.git/" || exit 1 From 43ab88164c28128b326bb99ccef0c0f9b1680141 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:14:56 -0400 Subject: [PATCH 023/110] fixup! t/perf/p7519: speed up test on Windows This reverts commit b34d252c709b1f39b562307254ab0e7d17956b6d. Signed-off-by: Jeff Hostetler --- t/perf/p7519-fsmonitor.sh | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/t/perf/p7519-fsmonitor.sh b/t/perf/p7519-fsmonitor.sh index 171644ffc90761..5eb5044a103cab 100755 --- a/t/perf/p7519-fsmonitor.sh +++ b/t/perf/p7519-fsmonitor.sh @@ -98,13 +98,6 @@ trace_stop() { fi } -touch_files() { - n=$1 - d="$n"_files - - (cd $d ; test_seq 1 $n | xargs touch ) -} - test_expect_success "one time repo setup" ' # set untrackedCache depending on the environment if test -n "$GIT_PERF_7519_UNTRACKED_CACHE" @@ -126,11 +119,10 @@ test_expect_success "one time repo setup" ' fi && mkdir 1_file 10_files 100_files 1000_files 10000_files && - touch_files 1 && - touch_files 10 && - touch_files 100 && - touch_files 1000 && - touch_files 10000 && + for i in $(test_seq 1 10); do touch 10_files/$i; done && + for i in $(test_seq 1 100); do touch 100_files/$i; done && + for i in $(test_seq 1 1000); do touch 1000_files/$i; done && + for i in $(test_seq 1 10000); do touch 10000_files/$i; done && git add 1_file 10_files 100_files 1000_files 10000_files && git commit -qm "Add files" && @@ -207,15 +199,15 @@ test_fsmonitor_suite() { # Update the mtimes on upto 100k files to make status think # that they are dirty. For simplicity, omit any files with - # LFs (i.e. anything that ls-files thinks it needs to dquote) - # and any files with whitespace so that they pass thru xargs - # properly. + # LFs (i.e. anything that ls-files thinks it needs to dquote). + # Then fully backslash-quote the paths to capture any + # whitespace so that they pass thru xargs properly. # test_perf_w_drop_caches "status (dirty) ($DESC)" ' git ls-files | \ head -100000 | \ grep -v \" | \ - egrep -v " ." | \ + sed '\''s/\(.\)/\\\1/g'\'' | \ xargs test-tool chmtime -300 && git status ' From b244b5416dcba34fc6c2fd78800e3ba3742b23aa Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:14:59 -0400 Subject: [PATCH 024/110] fixup! t/helper/test-chmtime: skip directories on Windows This reverts commit fd9522aa9ba9b6c3a637c6fb1c764ab35f756b4f. Signed-off-by: Jeff Hostetler --- t/helper/test-chmtime.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/t/helper/test-chmtime.c b/t/helper/test-chmtime.c index dc28890a183f1d..524b55ca496cab 100644 --- a/t/helper/test-chmtime.c +++ b/t/helper/test-chmtime.c @@ -134,21 +134,6 @@ int cmd__chmtime(int argc, const char **argv) } if (utb.modtime != sb.st_mtime && utime(argv[i], &utb) < 0) { -#ifdef GIT_WINDOWS_NATIVE - if (S_ISDIR(sb.st_mode)) { - /* - * NEEDSWORK: The Windows version of `utime()` - * (aka `mingw_utime()`) does not correctly - * handle directory arguments, since it uses - * `_wopen()`. Ignore it for now since this - * is just a test. - */ - fprintf(stderr, - ("Failed to modify time on directory %s. " - "Skipping\n"), argv[i]); - continue; - } -#endif fprintf(stderr, "Failed to modify time on %s: %s\n", argv[i], strerror(errno)); return 1; From 492c04ea7a7978cc7bff36ff5cb99cfb56035f1e Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:01 -0400 Subject: [PATCH 025/110] fixup! fsmonitor--daemon: implement handle_client callback This reverts commit d0ed68b4a1fa239f4c29d7030460646d0119b5f9. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 312 +----------------------------------- 1 file changed, 2 insertions(+), 310 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index d85157f56c0f4d..40b5e59ea58115 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -7,7 +7,6 @@ #include "fsmonitor--daemon.h" #include "simple-ipc.h" #include "khash.h" -#include "pkt-line.h" static const char * const builtin_fsmonitor__daemon_usage[] = { N_("git fsmonitor--daemon start []"), @@ -356,311 +355,6 @@ void fsmonitor_force_resync(struct fsmonitor_daemon_state *state) pthread_mutex_unlock(&state->main_lock); } -/* - * Format an opaque token string to send to the client. - */ -static void with_lock__format_response_token( - struct strbuf *response_token, - const struct strbuf *response_token_id, - const struct fsmonitor_batch *batch) -{ - /* assert current thread holding state->main_lock */ - - strbuf_reset(response_token); - strbuf_addf(response_token, "builtin:%s:%"PRIu64, - response_token_id->buf, batch->batch_seq_nr); -} - -/* - * Parse an opaque token from the client. - * Returns -1 on error. - */ -static int fsmonitor_parse_client_token(const char *buf_token, - struct strbuf *requested_token_id, - uint64_t *seq_nr) -{ - const char *p; - char *p_end; - - strbuf_reset(requested_token_id); - *seq_nr = 0; - - if (!skip_prefix(buf_token, "builtin:", &p)) - return -1; - - while (*p && *p != ':') - strbuf_addch(requested_token_id, *p++); - if (!*p++) - return -1; - - *seq_nr = (uint64_t)strtoumax(p, &p_end, 10); - if (*p_end) - return -1; - - return 0; -} - -KHASH_INIT(str, const char *, int, 0, kh_str_hash_func, kh_str_hash_equal); - -static int do_handle_client(struct fsmonitor_daemon_state *state, - const char *command, - ipc_server_reply_cb *reply, - struct ipc_server_reply_data *reply_data) -{ - struct fsmonitor_token_data *token_data = NULL; - struct strbuf response_token = STRBUF_INIT; - struct strbuf requested_token_id = STRBUF_INIT; - struct strbuf payload = STRBUF_INIT; - uint64_t requested_oldest_seq_nr = 0; - uint64_t total_response_len = 0; - const char *p; - const struct fsmonitor_batch *batch_head; - const struct fsmonitor_batch *batch; - intmax_t count = 0, duplicates = 0; - kh_str_t *shown; - int hash_ret; - int do_trivial = 0; - int do_flush = 0; - - /* - * We expect `command` to be of the form: - * - * := quit NUL - * | flush NUL - * | NUL - * | NUL - */ - - if (!strcmp(command, "quit")) { - /* - * A client has requested over the socket/pipe that the - * daemon shutdown. - * - * Tell the IPC thread pool to shutdown (which completes - * the await in the main thread (which can stop the - * fsmonitor listener thread)). - * - * There is no reply to the client. - */ - return SIMPLE_IPC_QUIT; - - } else if (!strcmp(command, "flush")) { - /* - * Flush all of our cached data and generate a new token - * just like if we lost sync with the filesystem. - * - * Then send a trivial response using the new token. - */ - do_flush = 1; - do_trivial = 1; - - } else if (!skip_prefix(command, "builtin:", &p)) { - /* assume V1 timestamp or garbage */ - - char *p_end; - - strtoumax(command, &p_end, 10); - trace_printf_key(&trace_fsmonitor, - ((*p_end) ? - "fsmonitor: invalid command line '%s'" : - "fsmonitor: unsupported V1 protocol '%s'"), - command); - do_trivial = 1; - - } else { - /* We have "builtin:*" */ - if (fsmonitor_parse_client_token(command, &requested_token_id, - &requested_oldest_seq_nr)) { - trace_printf_key(&trace_fsmonitor, - "fsmonitor: invalid V2 protocol token '%s'", - command); - do_trivial = 1; - - } else { - /* - * We have a V2 valid token: - * "builtin::" - */ - } - } - - pthread_mutex_lock(&state->main_lock); - - if (!state->current_token_data) - BUG("fsmonitor state does not have a current token"); - - if (do_flush) - with_lock__do_force_resync(state); - - /* - * We mark the current head of the batch list as "pinned" so - * that the listener thread will treat this item as read-only - * (and prevent any more paths from being added to it) from - * now on. - */ - token_data = state->current_token_data; - batch_head = token_data->batch_head; - ((struct fsmonitor_batch *)batch_head)->pinned_time = time(NULL); - - /* - * FSMonitor Protocol V2 requires that we send a response header - * with a "new current token" and then all of the paths that changed - * since the "requested token". We send the seq_nr of the just-pinned - * head batch so that future requests from a client will be relative - * to it. - */ - with_lock__format_response_token(&response_token, - &token_data->token_id, batch_head); - - reply(reply_data, response_token.buf, response_token.len + 1); - total_response_len += response_token.len + 1; - - trace2_data_string("fsmonitor", the_repository, "response/token", - response_token.buf); - trace_printf_key(&trace_fsmonitor, "response token: %s", - response_token.buf); - - if (!do_trivial) { - if (strcmp(requested_token_id.buf, token_data->token_id.buf)) { - /* - * The client last spoke to a different daemon - * instance -OR- the daemon had to resync with - * the filesystem (and lost events), so reject. - */ - trace2_data_string("fsmonitor", the_repository, - "response/token", "different"); - do_trivial = 1; - - } else if (requested_oldest_seq_nr < - token_data->batch_tail->batch_seq_nr) { - /* - * The client wants older events than we have for - * this token_id. This means that the end of our - * batch list was truncated and we cannot give the - * client a complete snapshot relative to their - * request. - */ - trace_printf_key(&trace_fsmonitor, - "client requested truncated data"); - do_trivial = 1; - } - } - - if (do_trivial) { - pthread_mutex_unlock(&state->main_lock); - - reply(reply_data, "/", 2); - - trace2_data_intmax("fsmonitor", the_repository, - "response/trivial", 1); - - strbuf_release(&response_token); - strbuf_release(&requested_token_id); - return 0; - } - - /* - * We're going to hold onto a pointer to the current - * token-data while we walk the list of batches of files. - * During this time, we will NOT be under the lock. - * So we ref-count it. - * - * This allows the listener thread to continue prepending - * new batches of items to the token-data (which we'll ignore). - * - * AND it allows the listener thread to do a token-reset - * (and install a new `current_token_data`). - */ - token_data->client_ref_count++; - - pthread_mutex_unlock(&state->main_lock); - - /* - * The client request is relative to the token that they sent, - * so walk the batch list backwards from the current head back - * to the batch (sequence number) they named. - * - * We use khash to de-dup the list of pathnames. - * - * NEEDSWORK: each batch contains a list of interned strings, - * so we only need to do pointer comparisons here to build the - * hash table. Currently, we're still comparing the string - * values. - */ - shown = kh_init_str(); - for (batch = batch_head; - batch && batch->batch_seq_nr > requested_oldest_seq_nr; - batch = batch->next) { - size_t k; - - for (k = 0; k < batch->nr; k++) { - const char *s = batch->interned_paths[k]; - size_t s_len; - - if (kh_get_str(shown, s) != kh_end(shown)) - duplicates++; - else { - kh_put_str(shown, s, &hash_ret); - - trace_printf_key(&trace_fsmonitor, - "send[%"PRIuMAX"]: %s", - count, s); - - /* Each path gets written with a trailing NUL */ - s_len = strlen(s) + 1; - - if (payload.len + s_len >= - LARGE_PACKET_DATA_MAX) { - reply(reply_data, payload.buf, - payload.len); - total_response_len += payload.len; - strbuf_reset(&payload); - } - - strbuf_add(&payload, s, s_len); - count++; - } - } - } - - if (payload.len) { - reply(reply_data, payload.buf, payload.len); - total_response_len += payload.len; - } - - kh_release_str(shown); - - pthread_mutex_lock(&state->main_lock); - - if (token_data->client_ref_count > 0) - token_data->client_ref_count--; - - if (token_data->client_ref_count == 0) { - if (token_data != state->current_token_data) { - /* - * The listener thread did a token-reset while we were - * walking the batch list. Therefore, this token is - * stale and can be discarded completely. If we are - * the last reader thread using this token, we own - * that work. - */ - fsmonitor_free_token_data(token_data); - } - } - - pthread_mutex_unlock(&state->main_lock); - - trace2_data_intmax("fsmonitor", the_repository, "response/length", total_response_len); - trace2_data_intmax("fsmonitor", the_repository, "response/count/files", count); - trace2_data_intmax("fsmonitor", the_repository, "response/count/duplicates", duplicates); - - strbuf_release(&response_token); - strbuf_release(&requested_token_id); - strbuf_release(&payload); - - return 0; -} - static ipc_server_application_cb handle_client; static int handle_client(void *data, @@ -668,7 +362,7 @@ static int handle_client(void *data, ipc_server_reply_cb *reply, struct ipc_server_reply_data *reply_data) { - struct fsmonitor_daemon_state *state = data; + /* struct fsmonitor_daemon_state *state = data; */ int result; /* @@ -679,12 +373,10 @@ static int handle_client(void *data, if (command_len != strlen(command)) BUG("FSMonitor assumes text messages"); - trace_printf_key(&trace_fsmonitor, "requested token: %s", command); - trace2_region_enter("fsmonitor", "handle_client", the_repository); trace2_data_string("fsmonitor", the_repository, "request", command); - result = do_handle_client(state, command, reply, reply_data); + result = 0; /* TODO Do something here. */ trace2_region_leave("fsmonitor", "handle_client", the_repository); From b27da829cdf186e9a694607fb02e4a6f5910a335 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:03 -0400 Subject: [PATCH 026/110] fixup! fsm-listen-darwin: implement FSEvent listener on MacOS This reverts commit 8657519ee51e4bb33e5732de21e45e5e6e1b8edd. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-listen-darwin.c | 381 --------------------------- 1 file changed, 381 deletions(-) diff --git a/compat/fsmonitor/fsm-listen-darwin.c b/compat/fsmonitor/fsm-listen-darwin.c index 83ae3609c1b44e..f424253d3eb037 100644 --- a/compat/fsmonitor/fsm-listen-darwin.c +++ b/compat/fsmonitor/fsm-listen-darwin.c @@ -97,401 +97,20 @@ void FSEventStreamRelease(FSEventStreamRef stream); #include "cache.h" #include "fsmonitor.h" #include "fsm-listen.h" -#include "fsmonitor--daemon.h" - -struct fsmonitor_daemon_backend_data -{ - CFStringRef cfsr_worktree_path; - CFStringRef cfsr_gitdir_path; - - CFArrayRef cfar_paths_to_watch; - int nr_paths_watching; - - FSEventStreamRef stream; - - CFRunLoopRef rl; - - enum shutdown_style { - SHUTDOWN_EVENT = 0, - FORCE_SHUTDOWN, - FORCE_ERROR_STOP, - } shutdown_style; - - unsigned int stream_scheduled:1; - unsigned int stream_started:1; -}; - -static void log_flags_set(const char *path, const FSEventStreamEventFlags flag) -{ - struct strbuf msg = STRBUF_INIT; - - if (flag & kFSEventStreamEventFlagMustScanSubDirs) - strbuf_addstr(&msg, "MustScanSubDirs|"); - if (flag & kFSEventStreamEventFlagUserDropped) - strbuf_addstr(&msg, "UserDropped|"); - if (flag & kFSEventStreamEventFlagKernelDropped) - strbuf_addstr(&msg, "KernelDropped|"); - if (flag & kFSEventStreamEventFlagEventIdsWrapped) - strbuf_addstr(&msg, "EventIdsWrapped|"); - if (flag & kFSEventStreamEventFlagHistoryDone) - strbuf_addstr(&msg, "HistoryDone|"); - if (flag & kFSEventStreamEventFlagRootChanged) - strbuf_addstr(&msg, "RootChanged|"); - if (flag & kFSEventStreamEventFlagMount) - strbuf_addstr(&msg, "Mount|"); - if (flag & kFSEventStreamEventFlagUnmount) - strbuf_addstr(&msg, "Unmount|"); - if (flag & kFSEventStreamEventFlagItemChangeOwner) - strbuf_addstr(&msg, "ItemChangeOwner|"); - if (flag & kFSEventStreamEventFlagItemCreated) - strbuf_addstr(&msg, "ItemCreated|"); - if (flag & kFSEventStreamEventFlagItemFinderInfoMod) - strbuf_addstr(&msg, "ItemFinderInfoMod|"); - if (flag & kFSEventStreamEventFlagItemInodeMetaMod) - strbuf_addstr(&msg, "ItemInodeMetaMod|"); - if (flag & kFSEventStreamEventFlagItemIsDir) - strbuf_addstr(&msg, "ItemIsDir|"); - if (flag & kFSEventStreamEventFlagItemIsFile) - strbuf_addstr(&msg, "ItemIsFile|"); - if (flag & kFSEventStreamEventFlagItemIsHardlink) - strbuf_addstr(&msg, "ItemIsHardlink|"); - if (flag & kFSEventStreamEventFlagItemIsLastHardlink) - strbuf_addstr(&msg, "ItemIsLastHardlink|"); - if (flag & kFSEventStreamEventFlagItemIsSymlink) - strbuf_addstr(&msg, "ItemIsSymlink|"); - if (flag & kFSEventStreamEventFlagItemModified) - strbuf_addstr(&msg, "ItemModified|"); - if (flag & kFSEventStreamEventFlagItemRemoved) - strbuf_addstr(&msg, "ItemRemoved|"); - if (flag & kFSEventStreamEventFlagItemRenamed) - strbuf_addstr(&msg, "ItemRenamed|"); - if (flag & kFSEventStreamEventFlagItemXattrMod) - strbuf_addstr(&msg, "ItemXattrMod|"); - if (flag & kFSEventStreamEventFlagOwnEvent) - strbuf_addstr(&msg, "OwnEvent|"); - if (flag & kFSEventStreamEventFlagItemCloned) - strbuf_addstr(&msg, "ItemCloned|"); - - trace_printf_key(&trace_fsmonitor, "fsevent: '%s', flags=%u %s", - path, flag, msg.buf); - - strbuf_release(&msg); -} - -static int ef_is_root_delete(const FSEventStreamEventFlags ef) -{ - return (ef & kFSEventStreamEventFlagItemIsDir && - ef & kFSEventStreamEventFlagItemRemoved); -} - -static int ef_is_root_renamed(const FSEventStreamEventFlags ef) -{ - return (ef & kFSEventStreamEventFlagItemIsDir && - ef & kFSEventStreamEventFlagItemRenamed); -} - -static int ef_is_dropped(const FSEventStreamEventFlags ef) -{ - return (ef & kFSEventStreamEventFlagKernelDropped || - ef & kFSEventStreamEventFlagUserDropped); -} - -static void fsevent_callback(ConstFSEventStreamRef streamRef, - void *ctx, - size_t num_of_events, - void *event_paths, - const FSEventStreamEventFlags event_flags[], - const FSEventStreamEventId event_ids[]) -{ - struct fsmonitor_daemon_state *state = ctx; - struct fsmonitor_daemon_backend_data *data = state->backend_data; - char **paths = (char **)event_paths; - struct fsmonitor_batch *batch = NULL; - struct string_list cookie_list = STRING_LIST_INIT_DUP; - const char *path_k; - const char *slash; - int k; - struct strbuf tmp = STRBUF_INIT; - - /* - * Build a list of all filesystem changes into a private/local - * list and without holding any locks. - */ - for (k = 0; k < num_of_events; k++) { - /* - * On Mac, we receive an array of absolute paths. - */ - path_k = paths[k]; - - /* - * If you want to debug FSEvents, log them to GIT_TRACE_FSMONITOR. - * Please don't log them to Trace2. - * - * trace_printf_key(&trace_fsmonitor, "Path: '%s'", path_k); - */ - - /* - * If event[k] is marked as dropped, we assume that we have - * lost sync with the filesystem and should flush our cached - * data. We need to: - * - * [1] Abort/wake any client threads waiting for a cookie and - * flush the cached state data (the current token), and - * create a new token. - * - * [2] Discard the batch that we were locally building (since - * they are conceptually relative to the just flushed - * token). - */ - if (ef_is_dropped(event_flags[k])) { - /* - * see also kFSEventStreamEventFlagMustScanSubDirs - */ - trace_printf_key(&trace_fsmonitor, "event: dropped"); - - fsmonitor_force_resync(state); - fsmonitor_batch__pop(batch); - string_list_clear(&cookie_list, 0); - - /* - * We assume that any events that we received - * in this callback after this dropped event - * may still be valid, so we continue rather - * than break. (And just in case there is a - * delete of ".git" hiding in there.) - */ - continue; - } - - switch (fsmonitor_classify_path_absolute(state, path_k)) { - - case IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX: - case IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX: - /* special case cookie files within .git or gitdir */ - - /* Use just the filename of the cookie file. */ - slash = find_last_dir_sep(path_k); - string_list_append(&cookie_list, - slash ? slash + 1 : path_k); - break; - - case IS_INSIDE_DOT_GIT: - case IS_INSIDE_GITDIR: - /* ignore all other paths inside of .git or gitdir */ - break; - - case IS_DOT_GIT: - case IS_GITDIR: - /* - * If .git directory is deleted or renamed away, - * we have to quit. - */ - if (ef_is_root_delete(event_flags[k])) { - trace_printf_key(&trace_fsmonitor, - "event: gitdir removed"); - goto force_shutdown; - } - if (ef_is_root_renamed(event_flags[k])) { - trace_printf_key(&trace_fsmonitor, - "event: gitdir renamed"); - goto force_shutdown; - } - break; - - case IS_WORKDIR_PATH: - /* try to queue normal pathnames */ - - if (trace_pass_fl(&trace_fsmonitor)) - log_flags_set(path_k, event_flags[k]); - - /* - * Because of the implicit "binning" (the - * kernel calls us at a given frequency) and - * de-duping (the kernel is free to combine - * multiple events for a given pathname), an - * individual fsevent could be marked as both - * a file and directory. Add it to the queue - * with both spellings so that the client will - * know how much to invalidate/refresh. - */ - - if (event_flags[k] & kFSEventStreamEventFlagItemIsFile) { - const char *rel = path_k + - state->path_worktree_watch.len + 1; - - if (!batch) - batch = fsmonitor_batch__new(); - fsmonitor_batch__add_path(batch, rel); - } - - if (event_flags[k] & kFSEventStreamEventFlagItemIsDir) { - const char *rel = path_k + - state->path_worktree_watch.len + 1; - - strbuf_reset(&tmp); - strbuf_addstr(&tmp, rel); - strbuf_addch(&tmp, '/'); - - if (!batch) - batch = fsmonitor_batch__new(); - fsmonitor_batch__add_path(batch, tmp.buf); - } - - break; - - case IS_OUTSIDE_CONE: - default: - trace_printf_key(&trace_fsmonitor, - "ignoring '%s'", path_k); - break; - } - } - - fsmonitor_publish(state, batch, &cookie_list); - string_list_clear(&cookie_list, 0); - strbuf_release(&tmp); - return; - -force_shutdown: - fsmonitor_batch__pop(batch); - string_list_clear(&cookie_list, 0); - - data->shutdown_style = FORCE_SHUTDOWN; - CFRunLoopStop(data->rl); - strbuf_release(&tmp); - return; -} - -/* - * NEEDSWORK: Investigate the proper value for the `latency` argument - * in the call to `FSEventStreamCreate()`. I'm not sure that this - * needs to be a config setting or just something that we tune after - * some testing. - * - * With a latency of 0.1, I was seeing lots of dropped events during - * the "touch 100000" files test within t/perf/p7519, but with a - * latency of 0.001 I did not see any dropped events. So the - * "correct" value may be somewhere in between. - * - * https://developer.apple.com/documentation/coreservices/1443980-fseventstreamcreate - */ int fsm_listen__ctor(struct fsmonitor_daemon_state *state) { - FSEventStreamCreateFlags flags = kFSEventStreamCreateFlagNoDefer | - kFSEventStreamCreateFlagWatchRoot | - kFSEventStreamCreateFlagFileEvents; - FSEventStreamContext ctx = { - 0, - state, - NULL, - NULL, - NULL - }; - struct fsmonitor_daemon_backend_data *data; - const void *dir_array[2]; - - CALLOC_ARRAY(data, 1); - state->backend_data = data; - - data->cfsr_worktree_path = CFStringCreateWithCString( - NULL, state->path_worktree_watch.buf, kCFStringEncodingUTF8); - dir_array[data->nr_paths_watching++] = data->cfsr_worktree_path; - - if (state->nr_paths_watching > 1) { - data->cfsr_gitdir_path = CFStringCreateWithCString( - NULL, state->path_gitdir_watch.buf, - kCFStringEncodingUTF8); - dir_array[data->nr_paths_watching++] = data->cfsr_gitdir_path; - } - - data->cfar_paths_to_watch = CFArrayCreate(NULL, dir_array, - data->nr_paths_watching, - NULL); - data->stream = FSEventStreamCreate(NULL, fsevent_callback, &ctx, - data->cfar_paths_to_watch, - kFSEventStreamEventIdSinceNow, - 0.001, flags); - if (data->stream == NULL) - goto failed; - - /* - * `data->rl` needs to be set inside the listener thread. - */ - - return 0; - -failed: - error("Unable to create FSEventStream."); - - FREE_AND_NULL(state->backend_data); return -1; } void fsm_listen__dtor(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data; - - if (!state || !state->backend_data) - return; - - data = state->backend_data; - - if (data->stream) { - if (data->stream_started) - FSEventStreamStop(data->stream); - if (data->stream_scheduled) - FSEventStreamInvalidate(data->stream); - FSEventStreamRelease(data->stream); - } - - FREE_AND_NULL(state->backend_data); } void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data; - - data = state->backend_data; - data->shutdown_style = SHUTDOWN_EVENT; - - CFRunLoopStop(data->rl); } void fsm_listen__loop(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data; - - data = state->backend_data; - - data->rl = CFRunLoopGetCurrent(); - - FSEventStreamScheduleWithRunLoop(data->stream, data->rl, kCFRunLoopDefaultMode); - data->stream_scheduled = 1; - - if (!FSEventStreamStart(data->stream)) { - error("Failed to start the FSEventStream"); - goto force_error_stop_without_loop; - } - data->stream_started = 1; - - CFRunLoopRun(); - - switch (data->shutdown_style) { - case FORCE_ERROR_STOP: - state->error_code = -1; - /* fall thru */ - case FORCE_SHUTDOWN: - ipc_server_stop_async(state->ipc_server_data); - /* fall thru */ - case SHUTDOWN_EVENT: - default: - break; - } - return; - -force_error_stop_without_loop: - state->error_code = -1; - ipc_server_stop_async(state->ipc_server_data); - return; } From 6fe4f740501b96d6f66f98580a3025d22242bfcf Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:08 -0400 Subject: [PATCH 027/110] fixup! fsm-listen-darwin: add macos header files for FSEvent This reverts commit ccc75e26a1460dfeede60847c705716843ce1994. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-listen-darwin.c | 96 ---------------------------- 1 file changed, 96 deletions(-) diff --git a/compat/fsmonitor/fsm-listen-darwin.c b/compat/fsmonitor/fsm-listen-darwin.c index f424253d3eb037..c84e3344ab99b4 100644 --- a/compat/fsmonitor/fsm-listen-darwin.c +++ b/compat/fsmonitor/fsm-listen-darwin.c @@ -1,99 +1,3 @@ -#if defined(__GNUC__) -/* - * It is possible to #include CoreFoundation/CoreFoundation.h when compiling - * with clang, but not with GCC as of time of writing. - * - * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93082 for details. - */ -typedef unsigned int FSEventStreamCreateFlags; -#define kFSEventStreamEventFlagNone 0x00000000 -#define kFSEventStreamEventFlagMustScanSubDirs 0x00000001 -#define kFSEventStreamEventFlagUserDropped 0x00000002 -#define kFSEventStreamEventFlagKernelDropped 0x00000004 -#define kFSEventStreamEventFlagEventIdsWrapped 0x00000008 -#define kFSEventStreamEventFlagHistoryDone 0x00000010 -#define kFSEventStreamEventFlagRootChanged 0x00000020 -#define kFSEventStreamEventFlagMount 0x00000040 -#define kFSEventStreamEventFlagUnmount 0x00000080 -#define kFSEventStreamEventFlagItemCreated 0x00000100 -#define kFSEventStreamEventFlagItemRemoved 0x00000200 -#define kFSEventStreamEventFlagItemInodeMetaMod 0x00000400 -#define kFSEventStreamEventFlagItemRenamed 0x00000800 -#define kFSEventStreamEventFlagItemModified 0x00001000 -#define kFSEventStreamEventFlagItemFinderInfoMod 0x00002000 -#define kFSEventStreamEventFlagItemChangeOwner 0x00004000 -#define kFSEventStreamEventFlagItemXattrMod 0x00008000 -#define kFSEventStreamEventFlagItemIsFile 0x00010000 -#define kFSEventStreamEventFlagItemIsDir 0x00020000 -#define kFSEventStreamEventFlagItemIsSymlink 0x00040000 -#define kFSEventStreamEventFlagOwnEvent 0x00080000 -#define kFSEventStreamEventFlagItemIsHardlink 0x00100000 -#define kFSEventStreamEventFlagItemIsLastHardlink 0x00200000 -#define kFSEventStreamEventFlagItemCloned 0x00400000 - -typedef struct __FSEventStream *FSEventStreamRef; -typedef const FSEventStreamRef ConstFSEventStreamRef; - -typedef unsigned int CFStringEncoding; -#define kCFStringEncodingUTF8 0x08000100 - -typedef const struct __CFString *CFStringRef; -typedef const struct __CFArray *CFArrayRef; -typedef const struct __CFRunLoop *CFRunLoopRef; - -struct FSEventStreamContext { - long long version; - void *cb_data, *retain, *release, *copy_description; -}; - -typedef struct FSEventStreamContext FSEventStreamContext; -typedef unsigned int FSEventStreamEventFlags; -#define kFSEventStreamCreateFlagNoDefer 0x02 -#define kFSEventStreamCreateFlagWatchRoot 0x04 -#define kFSEventStreamCreateFlagFileEvents 0x10 - -typedef unsigned long long FSEventStreamEventId; -#define kFSEventStreamEventIdSinceNow 0xFFFFFFFFFFFFFFFFULL - -typedef void (*FSEventStreamCallback)(ConstFSEventStreamRef streamRef, - void *context, - __SIZE_TYPE__ num_of_events, - void *event_paths, - const FSEventStreamEventFlags event_flags[], - const FSEventStreamEventId event_ids[]); -typedef double CFTimeInterval; -FSEventStreamRef FSEventStreamCreate(void *allocator, - FSEventStreamCallback callback, - FSEventStreamContext *context, - CFArrayRef paths_to_watch, - FSEventStreamEventId since_when, - CFTimeInterval latency, - FSEventStreamCreateFlags flags); -CFStringRef CFStringCreateWithCString(void *allocator, const char *string, - CFStringEncoding encoding); -CFArrayRef CFArrayCreate(void *allocator, const void **items, long long count, - void *callbacks); -void CFRunLoopRun(void); -void CFRunLoopStop(CFRunLoopRef run_loop); -CFRunLoopRef CFRunLoopGetCurrent(void); -extern CFStringRef kCFRunLoopDefaultMode; -void FSEventStreamScheduleWithRunLoop(FSEventStreamRef stream, - CFRunLoopRef run_loop, - CFStringRef run_loop_mode); -unsigned char FSEventStreamStart(FSEventStreamRef stream); -void FSEventStreamStop(FSEventStreamRef stream); -void FSEventStreamInvalidate(FSEventStreamRef stream); -void FSEventStreamRelease(FSEventStreamRef stream); -#else -/* - * Let Apple's headers declare `isalnum()` first, before - * Git's headers override it via a constant - */ -#include -#include -#include -#endif - #include "cache.h" #include "fsmonitor.h" #include "fsm-listen.h" From 32c3a207fcf201fcee3afbc44795c02b6797746e Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:11 -0400 Subject: [PATCH 028/110] fixup! fsm-listen-win32: implement FSMonitor backend on Windows This reverts commit a25a759709afea331f533a68fed190afc68ab07f. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-listen-win32.c | 561 ---------------------------- 1 file changed, 561 deletions(-) diff --git a/compat/fsmonitor/fsm-listen-win32.c b/compat/fsmonitor/fsm-listen-win32.c index 5cd4d136e64441..916cbea254f8cf 100644 --- a/compat/fsmonitor/fsm-listen-win32.c +++ b/compat/fsmonitor/fsm-listen-win32.c @@ -2,581 +2,20 @@ #include "config.h" #include "fsmonitor.h" #include "fsm-listen.h" -#include "fsmonitor--daemon.h" - -/* - * The documentation of ReadDirectoryChangesW() states that the maximum - * buffer size is 64K when the monitored directory is remote. - * - * Larger buffers may be used when the monitored directory is local and - * will help us receive events faster from the kernel and avoid dropped - * events. - * - * So we try to use a very large buffer and silently fallback to 64K if - * we get an error. - */ -#define MAX_RDCW_BUF_FALLBACK (65536) -#define MAX_RDCW_BUF (65536 * 8) - -struct one_watch -{ - char buffer[MAX_RDCW_BUF]; - DWORD buf_len; - DWORD count; - - struct strbuf path; - HANDLE hDir; - HANDLE hEvent; - OVERLAPPED overlapped; - - /* - * Is there an active ReadDirectoryChangesW() call pending. If so, we - * need to later call GetOverlappedResult() and possibly CancelIoEx(). - */ - BOOL is_active; -}; - -struct fsmonitor_daemon_backend_data -{ - struct one_watch *watch_worktree; - struct one_watch *watch_gitdir; - - HANDLE hEventShutdown; - - HANDLE hListener[3]; /* we don't own these handles */ -#define LISTENER_SHUTDOWN 0 -#define LISTENER_HAVE_DATA_WORKTREE 1 -#define LISTENER_HAVE_DATA_GITDIR 2 - int nr_listener_handles; -}; - -/* - * Convert the WCHAR path from the notification into UTF8 and - * then normalize it. - */ -static int normalize_path_in_utf8(FILE_NOTIFY_INFORMATION *info, - struct strbuf *normalized_path) -{ - int reserve; - int len = 0; - - strbuf_reset(normalized_path); - if (!info->FileNameLength) - goto normalize; - - /* - * Pre-reserve enough space in the UTF8 buffer for - * each Unicode WCHAR character to be mapped into a - * sequence of 2 UTF8 characters. That should let us - * avoid ERROR_INSUFFICIENT_BUFFER 99.9+% of the time. - */ - reserve = info->FileNameLength + 1; - strbuf_grow(normalized_path, reserve); - - for (;;) { - len = WideCharToMultiByte(CP_UTF8, 0, info->FileName, - info->FileNameLength / sizeof(WCHAR), - normalized_path->buf, - strbuf_avail(normalized_path) - 1, - NULL, NULL); - if (len > 0) - goto normalize; - if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) { - error("[GLE %ld] could not convert path to UTF-8: '%.*ls'", - GetLastError(), - (int)(info->FileNameLength / sizeof(WCHAR)), - info->FileName); - return -1; - } - - strbuf_grow(normalized_path, - strbuf_avail(normalized_path) + reserve); - } - -normalize: - strbuf_setlen(normalized_path, len); - return strbuf_normalize_path(normalized_path); -} void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) { - SetEvent(state->backend_data->hListener[LISTENER_SHUTDOWN]); -} - -static struct one_watch *create_watch(struct fsmonitor_daemon_state *state, - const char *path) -{ - struct one_watch *watch = NULL; - DWORD desired_access = FILE_LIST_DIRECTORY; - DWORD share_mode = - FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE; - HANDLE hDir; - wchar_t wpath[MAX_PATH]; - - if (xutftowcs_path(wpath, path) < 0) { - error(_("could not convert to wide characters: '%s'"), path); - return NULL; - } - - hDir = CreateFileW(wpath, - desired_access, share_mode, NULL, OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED, - NULL); - if (hDir == INVALID_HANDLE_VALUE) { - error(_("[GLE %ld] could not watch '%s'"), - GetLastError(), path); - return NULL; - } - - CALLOC_ARRAY(watch, 1); - - watch->buf_len = sizeof(watch->buffer); /* assume full MAX_RDCW_BUF */ - - strbuf_init(&watch->path, 0); - strbuf_addstr(&watch->path, path); - - watch->hDir = hDir; - watch->hEvent = CreateEvent(NULL, TRUE, FALSE, NULL); - - return watch; -} - -static void destroy_watch(struct one_watch *watch) -{ - if (!watch) - return; - - strbuf_release(&watch->path); - if (watch->hDir != INVALID_HANDLE_VALUE) - CloseHandle(watch->hDir); - if (watch->hEvent != INVALID_HANDLE_VALUE) - CloseHandle(watch->hEvent); - - free(watch); -} - -static int start_rdcw_watch(struct fsmonitor_daemon_backend_data *data, - struct one_watch *watch) -{ - DWORD dwNotifyFilter = - FILE_NOTIFY_CHANGE_FILE_NAME | - FILE_NOTIFY_CHANGE_DIR_NAME | - FILE_NOTIFY_CHANGE_ATTRIBUTES | - FILE_NOTIFY_CHANGE_SIZE | - FILE_NOTIFY_CHANGE_LAST_WRITE | - FILE_NOTIFY_CHANGE_CREATION; - - ResetEvent(watch->hEvent); - - memset(&watch->overlapped, 0, sizeof(watch->overlapped)); - watch->overlapped.hEvent = watch->hEvent; - - /* - * Queue an async call using Overlapped IO. This returns immediately. - * Our event handle will be signalled when the real result is available. - * - * The return value here just means that we successfully queued it. - * We won't know if the Read...() actually produces data until later. - */ - watch->is_active = ReadDirectoryChangesW( - watch->hDir, watch->buffer, watch->buf_len, TRUE, - dwNotifyFilter, &watch->count, &watch->overlapped, NULL); - - if (watch->is_active) - return 0; - - error("ReadDirectoryChangedW failed on '%s' [GLE %ld]", - watch->path.buf, GetLastError()); - return -1; -} - -static int recv_rdcw_watch(struct one_watch *watch) -{ - DWORD gle; - - watch->is_active = FALSE; - - /* - * The overlapped result is ready. If the Read...() was successful - * we finally receive the actual result into our buffer. - */ - if (GetOverlappedResult(watch->hDir, &watch->overlapped, &watch->count, - TRUE)) - return 0; - - gle = GetLastError(); - if (gle == ERROR_INVALID_PARAMETER && - /* - * The kernel throws an invalid parameter error when our - * buffer is too big and we are pointed at a remote - * directory (and possibly for other reasons). Quietly - * set it down and try again. - * - * See note about MAX_RDCW_BUF at the top. - */ - watch->buf_len > MAX_RDCW_BUF_FALLBACK) { - watch->buf_len = MAX_RDCW_BUF_FALLBACK; - return -2; - } - - /* - * NEEDSWORK: If an external is deleted, the above - * returns an error. I'm not sure that there's anything that - * we can do here other than failing -- the /.git - * link file would be broken anyway. We might try to check - * for that and return a better error message, but I'm not - * sure it is worth it. - */ - - error("GetOverlappedResult failed on '%s' [GLE %ld]", - watch->path.buf, gle); - return -1; -} - -static void cancel_rdcw_watch(struct one_watch *watch) -{ - DWORD count; - - if (!watch || !watch->is_active) - return; - - /* - * The calls to ReadDirectoryChangesW() and GetOverlappedResult() - * form a "pair" (my term) where we queue an IO and promise to - * hang around and wait for the kernel to give us the result. - * - * If for some reason after we queue the IO, we have to quit - * or otherwise not stick around for the second half, we must - * tell the kernel to abort the IO. This prevents the kernel - * from writing to our buffer and/or signalling our event - * after we free them. - * - * (Ask me how much fun it was to track that one down). - */ - CancelIoEx(watch->hDir, &watch->overlapped); - GetOverlappedResult(watch->hDir, &watch->overlapped, &count, TRUE); - watch->is_active = FALSE; -} - -/* - * Process filesystem events that happen anywhere (recursively) under the - * root directory. For a normal working directory, this includes - * both version controlled files and the contents of the .git/ directory. - * - * If /.git is a file, then we only see events for the file - * itself. - */ -static int process_worktree_events(struct fsmonitor_daemon_state *state) -{ - struct fsmonitor_daemon_backend_data *data = state->backend_data; - struct one_watch *watch = data->watch_worktree; - struct strbuf path = STRBUF_INIT; - struct string_list cookie_list = STRING_LIST_INIT_DUP; - struct fsmonitor_batch *batch = NULL; - const char *p = watch->buffer; - - /* - * If the kernel gets more events than will fit in the kernel - * buffer associated with our RDCW handle, it drops them and - * returns a count of zero. - * - * Yes, the call returns WITHOUT error and with length zero. - * - * (The "overflow" case is not ambiguous with the "no data" case - * because we did an INFINITE wait.) - * - * This means we have a gap in coverage. Tell the daemon layer - * to resync. - */ - if (!watch->count) { - trace2_data_string("fsmonitor", NULL, "fsm-listen/kernel", - "overflow"); - fsmonitor_force_resync(state); - return LISTENER_HAVE_DATA_WORKTREE; - } - - /* - * On Windows, `info` contains an "array" of paths that are - * relative to the root of whichever directory handle received - * the event. - */ - for (;;) { - FILE_NOTIFY_INFORMATION *info = (void *)p; - const char *slash; - enum fsmonitor_path_type t; - - strbuf_reset(&path); - if (normalize_path_in_utf8(info, &path) == -1) - goto skip_this_path; - - t = fsmonitor_classify_path_workdir_relative(path.buf); - - switch (t) { - case IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX: - /* special case cookie files within .git */ - - /* Use just the filename of the cookie file. */ - slash = find_last_dir_sep(path.buf); - string_list_append(&cookie_list, - slash ? slash + 1 : path.buf); - break; - - case IS_INSIDE_DOT_GIT: - /* ignore everything inside of "/.git/" */ - break; - - case IS_DOT_GIT: - /* "/.git" was deleted (or renamed away) */ - if ((info->Action == FILE_ACTION_REMOVED) || - (info->Action == FILE_ACTION_RENAMED_OLD_NAME)) { - trace2_data_string("fsmonitor", NULL, - "fsm-listen/dotgit", - "removed"); - goto force_shutdown; - } - break; - - case IS_WORKDIR_PATH: - /* queue normal pathname */ - if (!batch) - batch = fsmonitor_batch__new(); - fsmonitor_batch__add_path(batch, path.buf); - break; - - case IS_GITDIR: - case IS_INSIDE_GITDIR: - case IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX: - default: - BUG("unexpected path classification '%d' for '%s'", - t, path.buf); - } - -skip_this_path: - if (!info->NextEntryOffset) - break; - p += info->NextEntryOffset; - } - - fsmonitor_publish(state, batch, &cookie_list); - batch = NULL; - string_list_clear(&cookie_list, 0); - strbuf_release(&path); - return LISTENER_HAVE_DATA_WORKTREE; - -force_shutdown: - fsmonitor_batch__pop(batch); - string_list_clear(&cookie_list, 0); - strbuf_release(&path); - return LISTENER_SHUTDOWN; -} - -/* - * Process filesystem events that happened anywhere (recursively) under the - * external (such as non-primary worktrees or submodules). - * We only care about cookie files that our client threads created here. - * - * Note that we DO NOT get filesystem events on the external - * itself (it is not inside something that we are watching). In particular, - * we do not get an event if the external is deleted. - */ -static int process_gitdir_events(struct fsmonitor_daemon_state *state) -{ - struct fsmonitor_daemon_backend_data *data = state->backend_data; - struct one_watch *watch = data->watch_gitdir; - struct strbuf path = STRBUF_INIT; - struct string_list cookie_list = STRING_LIST_INIT_DUP; - const char *p = watch->buffer; - - if (!watch->count) { - trace2_data_string("fsmonitor", NULL, "fsm-listen/kernel", - "overflow"); - fsmonitor_force_resync(state); - return LISTENER_HAVE_DATA_GITDIR; - } - - for (;;) { - FILE_NOTIFY_INFORMATION *info = (void *)p; - const char *slash; - enum fsmonitor_path_type t; - - strbuf_reset(&path); - if (normalize_path_in_utf8(info, &path) == -1) - goto skip_this_path; - - t = fsmonitor_classify_path_gitdir_relative(path.buf); - - switch (t) { - case IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX: - /* special case cookie files within gitdir */ - - /* Use just the filename of the cookie file. */ - slash = find_last_dir_sep(path.buf); - string_list_append(&cookie_list, - slash ? slash + 1 : path.buf); - break; - - case IS_INSIDE_GITDIR: - goto skip_this_path; - - default: - BUG("unexpected path classification '%d' for '%s'", - t, path.buf); - } - -skip_this_path: - if (!info->NextEntryOffset) - break; - p += info->NextEntryOffset; - } - - fsmonitor_publish(state, NULL, &cookie_list); - string_list_clear(&cookie_list, 0); - strbuf_release(&path); - return LISTENER_HAVE_DATA_GITDIR; } void fsm_listen__loop(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data = state->backend_data; - DWORD dwWait; - int result; - - state->error_code = 0; - - if (start_rdcw_watch(data, data->watch_worktree) == -1) - goto force_error_stop; - - if (data->watch_gitdir && - start_rdcw_watch(data, data->watch_gitdir) == -1) - goto force_error_stop; - - for (;;) { - dwWait = WaitForMultipleObjects(data->nr_listener_handles, - data->hListener, - FALSE, INFINITE); - - if (dwWait == WAIT_OBJECT_0 + LISTENER_HAVE_DATA_WORKTREE) { - result = recv_rdcw_watch(data->watch_worktree); - if (result == -1) { - /* hard error */ - goto force_error_stop; - } - if (result == -2) { - /* retryable error */ - if (start_rdcw_watch(data, data->watch_worktree) == -1) - goto force_error_stop; - continue; - } - - /* have data */ - if (process_worktree_events(state) == LISTENER_SHUTDOWN) - goto force_shutdown; - if (start_rdcw_watch(data, data->watch_worktree) == -1) - goto force_error_stop; - continue; - } - - if (dwWait == WAIT_OBJECT_0 + LISTENER_HAVE_DATA_GITDIR) { - result = recv_rdcw_watch(data->watch_gitdir); - if (result == -1) { - /* hard error */ - goto force_error_stop; - } - if (result == -2) { - /* retryable error */ - if (start_rdcw_watch(data, data->watch_gitdir) == -1) - goto force_error_stop; - continue; - } - - /* have data */ - if (process_gitdir_events(state) == LISTENER_SHUTDOWN) - goto force_shutdown; - if (start_rdcw_watch(data, data->watch_gitdir) == -1) - goto force_error_stop; - continue; - } - - if (dwWait == WAIT_OBJECT_0 + LISTENER_SHUTDOWN) - goto clean_shutdown; - - error(_("could not read directory changes [GLE %ld]"), - GetLastError()); - goto force_error_stop; - } - -force_error_stop: - state->error_code = -1; - -force_shutdown: - /* - * Tell the IPC thead pool to stop (which completes the await - * in the main thread (which will also signal this thread (if - * we are still alive))). - */ - ipc_server_stop_async(state->ipc_server_data); - -clean_shutdown: - cancel_rdcw_watch(data->watch_worktree); - cancel_rdcw_watch(data->watch_gitdir); } int fsm_listen__ctor(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data; - - CALLOC_ARRAY(data, 1); - - data->hEventShutdown = CreateEvent(NULL, TRUE, FALSE, NULL); - - data->watch_worktree = create_watch(state, - state->path_worktree_watch.buf); - if (!data->watch_worktree) - goto failed; - - if (state->nr_paths_watching > 1) { - data->watch_gitdir = create_watch(state, - state->path_gitdir_watch.buf); - if (!data->watch_gitdir) - goto failed; - } - - data->hListener[LISTENER_SHUTDOWN] = data->hEventShutdown; - data->nr_listener_handles++; - - data->hListener[LISTENER_HAVE_DATA_WORKTREE] = - data->watch_worktree->hEvent; - data->nr_listener_handles++; - - if (data->watch_gitdir) { - data->hListener[LISTENER_HAVE_DATA_GITDIR] = - data->watch_gitdir->hEvent; - data->nr_listener_handles++; - } - - state->backend_data = data; - return 0; - -failed: - CloseHandle(data->hEventShutdown); - destroy_watch(data->watch_worktree); - destroy_watch(data->watch_gitdir); - return -1; } void fsm_listen__dtor(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data; - - if (!state || !state->backend_data) - return; - - data = state->backend_data; - - CloseHandle(data->hEventShutdown); - destroy_watch(data->watch_worktree); - destroy_watch(data->watch_gitdir); - - FREE_AND_NULL(state->backend_data); } From f394cb19737d19e99e08eb10ee77623cc5ebc6be Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:13 -0400 Subject: [PATCH 029/110] fixup! fsmonitor--daemon: create token-based changed path cache This reverts commit 68a4ae60326de2c0cff4c3dde9b56aa59b6f0fc2. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 234 +----------------------------------- fsmonitor--daemon.h | 40 ------ 2 files changed, 2 insertions(+), 272 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 40b5e59ea58115..ecf8d5d3200368 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -168,27 +168,17 @@ struct fsmonitor_token_data { uint64_t client_ref_count; }; -struct fsmonitor_batch { - struct fsmonitor_batch *next; - uint64_t batch_seq_nr; - const char **interned_paths; - size_t nr, alloc; - time_t pinned_time; -}; - static struct fsmonitor_token_data *fsmonitor_new_token_data(void) { static int test_env_value = -1; static uint64_t flush_count = 0; struct fsmonitor_token_data *token; - struct fsmonitor_batch *batch; CALLOC_ARRAY(token, 1); - batch = fsmonitor_batch__new(); strbuf_init(&token->token_id, 0); - token->batch_head = batch; - token->batch_tail = batch; + token->batch_head = NULL; + token->batch_tail = NULL; token->client_ref_count = 0; if (test_env_value < 0) @@ -214,147 +204,9 @@ static struct fsmonitor_token_data *fsmonitor_new_token_data(void) strbuf_addf(&token->token_id, "test_%08x", test_env_value++); } - /* - * We created a new and are starting a new series - * of tokens with a zero . - * - * Since clients cannot guess our new (non test) - * they will always receive a trivial response (because of the - * mismatch on the ). The trivial response will - * tell them our new so that subsequent requests - * will be relative to our new series. (And when sending that - * response, we pin the current head of the batch list.) - * - * Even if the client correctly guesses the , their - * request of "builtin::0" asks for all changes MORE - * RECENT than batch/bin 0. - * - * This implies that it is a waste to accumulate paths in the - * initial batch/bin (because they will never be transmitted). - * - * So the daemon could be running for days and watching the - * file system, but doesn't need to actually accumulate any - * paths UNTIL we need to set a reference point for a later - * relative request. - * - * However, it is very useful for testing to always have a - * reference point set. Pin batch 0 to force early file system - * events to accumulate. - */ - if (test_env_value) - batch->pinned_time = time(NULL); - return token; } -struct fsmonitor_batch *fsmonitor_batch__new(void) -{ - struct fsmonitor_batch *batch; - - CALLOC_ARRAY(batch, 1); - - return batch; -} - -struct fsmonitor_batch *fsmonitor_batch__pop(struct fsmonitor_batch *batch) -{ - struct fsmonitor_batch *next; - - if (!batch) - return NULL; - - next = batch->next; - - /* - * The actual strings within the array are interned, so we don't - * own them. - */ - free(batch->interned_paths); - - return next; -} - -void fsmonitor_batch__add_path(struct fsmonitor_batch *batch, - const char *path) -{ - const char *interned_path = strintern(path); - - trace_printf_key(&trace_fsmonitor, "event: %s", interned_path); - - ALLOC_GROW(batch->interned_paths, batch->nr + 1, batch->alloc); - batch->interned_paths[batch->nr++] = interned_path; -} - -static void fsmonitor_batch__combine(struct fsmonitor_batch *batch_dest, - const struct fsmonitor_batch *batch_src) -{ - size_t k; - - ALLOC_GROW(batch_dest->interned_paths, - batch_dest->nr + batch_src->nr + 1, - batch_dest->alloc); - - for (k = 0; k < batch_src->nr; k++) - batch_dest->interned_paths[batch_dest->nr++] = - batch_src->interned_paths[k]; -} - -static void fsmonitor_free_token_data(struct fsmonitor_token_data *token) -{ - struct fsmonitor_batch *p; - - if (!token) - return; - - assert(token->client_ref_count == 0); - - strbuf_release(&token->token_id); - - for (p = token->batch_head; p; p = fsmonitor_batch__pop(p)) - ; - - free(token); -} - -/* - * Flush all of our cached data about the filesystem. Call this if we - * lose sync with the filesystem and miss some notification events. - * - * [1] If we are missing events, then we no longer have a complete - * history of the directory (relative to our current start token). - * We should create a new token and start fresh (as if we just - * booted up). - * - * If there are no concurrent threads readering the current token data - * series, we can free it now. Otherwise, let the last reader free - * it. - * - * Either way, the old token data series is no longer associated with - * our state data. - */ -static void with_lock__do_force_resync(struct fsmonitor_daemon_state *state) -{ - /* assert current thread holding state->main_lock */ - - struct fsmonitor_token_data *free_me = NULL; - struct fsmonitor_token_data *new_one = NULL; - - new_one = fsmonitor_new_token_data(); - - if (state->current_token_data->client_ref_count == 0) - free_me = state->current_token_data; - state->current_token_data = new_one; - - fsmonitor_free_token_data(free_me); -} - -void fsmonitor_force_resync(struct fsmonitor_daemon_state *state) -{ - pthread_mutex_lock(&state->main_lock); - with_lock__do_force_resync(state); - pthread_mutex_unlock(&state->main_lock); -} - static ipc_server_application_cb handle_client; static int handle_client(void *data, @@ -464,81 +316,6 @@ enum fsmonitor_path_type fsmonitor_classify_path_absolute( return fsmonitor_classify_path_gitdir_relative(rel); } -/* - * We try to combine small batches at the front of the batch-list to avoid - * having a long list. This hopefully makes it a little easier when we want - * to truncate and maintain the list. However, we don't want the paths array - * to just keep growing and growing with realloc, so we insert an arbitrary - * limit. - */ -#define MY_COMBINE_LIMIT (1024) - -void fsmonitor_publish(struct fsmonitor_daemon_state *state, - struct fsmonitor_batch *batch, - const struct string_list *cookie_names) -{ - if (!batch && !cookie_names->nr) - return; - - pthread_mutex_lock(&state->main_lock); - - if (batch) { - struct fsmonitor_batch *head; - - head = state->current_token_data->batch_head; - if (!head) { - BUG("token does not have batch"); - } else if (head->pinned_time) { - /* - * We cannot alter the current batch list - * because: - * - * [a] it is being transmitted to at least one - * client and the handle_client() thread has a - * ref-count, but not a lock on the batch list - * starting with this item. - * - * [b] it has been transmitted in the past to - * at least one client such that future - * requests are relative to this head batch. - * - * So, we can only prepend a new batch onto - * the front of the list. - */ - batch->batch_seq_nr = head->batch_seq_nr + 1; - batch->next = head; - state->current_token_data->batch_head = batch; - } else if (!head->batch_seq_nr) { - /* - * Batch 0 is unpinned. See the note in - * `fsmonitor_new_token_data()` about why we - * don't need to accumulate these paths. - */ - fsmonitor_batch__pop(batch); - } else if (head->nr + batch->nr > MY_COMBINE_LIMIT) { - /* - * The head batch in the list has never been - * transmitted to a client, but folding the - * contents of the new batch onto it would - * exceed our arbitrary limit, so just prepend - * the new batch onto the list. - */ - batch->batch_seq_nr = head->batch_seq_nr + 1; - batch->next = head; - state->current_token_data->batch_head = batch; - } else { - /* - * We are free to append the paths in the given - * batch onto the end of the current head batch. - */ - fsmonitor_batch__combine(head, batch); - fsmonitor_batch__pop(batch); - } - } - - pthread_mutex_unlock(&state->main_lock); -} - static void *fsm_listen__thread_proc(void *_state) { struct fsmonitor_daemon_state *state = _state; @@ -553,13 +330,6 @@ static void *fsm_listen__thread_proc(void *_state) fsm_listen__loop(state); - pthread_mutex_lock(&state->main_lock); - if (state->current_token_data && - state->current_token_data->client_ref_count == 0) - fsmonitor_free_token_data(state->current_token_data); - state->current_token_data = NULL; - pthread_mutex_unlock(&state->main_lock); - trace2_thread_exit(); return NULL; } diff --git a/fsmonitor--daemon.h b/fsmonitor--daemon.h index 89a9ef20b24b45..7bbb3a27a1ce1d 100644 --- a/fsmonitor--daemon.h +++ b/fsmonitor--daemon.h @@ -12,27 +12,6 @@ struct fsmonitor_batch; struct fsmonitor_token_data; -/* - * Create a new batch of path(s). The returned batch is considered - * private and not linked into the fsmonitor daemon state. The caller - * should fill this batch with one or more paths and then publish it. - */ -struct fsmonitor_batch *fsmonitor_batch__new(void); - -/* - * Free this batch and return the value of the batch->next field. - */ -struct fsmonitor_batch *fsmonitor_batch__pop(struct fsmonitor_batch *batch); - -/* - * Add this path to this batch of modified files. - * - * The batch should be private and NOT (yet) linked into the fsmonitor - * daemon state and therefore not yet visible to worker threads and so - * no locking is required. - */ -void fsmonitor_batch__add_path(struct fsmonitor_batch *batch, const char *path); - struct fsmonitor_daemon_backend_data; /* opaque platform-specific data */ struct fsmonitor_daemon_state { @@ -112,24 +91,5 @@ enum fsmonitor_path_type fsmonitor_classify_path_absolute( struct fsmonitor_daemon_state *state, const char *path); -/* - * Prepend the this batch of path(s) onto the list of batches associated - * with the current token. This makes the batch visible to worker threads. - * - * The caller no longer owns the batch and must not free it. - * - * Wake up the client threads waiting on these cookies. - */ -void fsmonitor_publish(struct fsmonitor_daemon_state *state, - struct fsmonitor_batch *batch, - const struct string_list *cookie_names); - -/* - * If the platform-specific layer loses sync with the filesystem, - * it should call this to invalidate cached data and abort waiting - * threads. - */ -void fsmonitor_force_resync(struct fsmonitor_daemon_state *state); - #endif /* HAVE_FSMONITOR_DAEMON_BACKEND */ #endif /* FSMONITOR_DAEMON_H */ From 1be0de12a38f15d477721c3b9a01b82650a0672b Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:15 -0400 Subject: [PATCH 030/110] fixup! fsmonitor--daemon: define token-ids This reverts commit 13cf2b3944723e196f47aa82c50b2e9059f7e80a. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 116 +----------------------------------- 1 file changed, 1 insertion(+), 115 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index ecf8d5d3200368..2f89452d93b55d 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -93,120 +93,6 @@ static int do_as_client__status(void) } } -/* - * Requests to and from a FSMonitor Protocol V2 provider use an opaque - * "token" as a virtual timestamp. Clients can request a summary of all - * created/deleted/modified files relative to a token. In the response, - * clients receive a new token for the next (relative) request. - * - * - * Token Format - * ============ - * - * The contents of the token are private and provider-specific. - * - * For the built-in fsmonitor--daemon, we define a token as follows: - * - * "builtin" ":" ":" - * - * The "builtin" prefix is used as a namespace to avoid conflicts - * with other providers (such as Watchman). - * - * The is an arbitrary OPAQUE string, such as a GUID, - * UUID, or {timestamp,pid}. It is used to group all filesystem - * events that happened while the daemon was monitoring (and in-sync - * with the filesystem). - * - * Unlike FSMonitor Protocol V1, it is not defined as a timestamp - * and does not define less-than/greater-than relationships. - * (There are too many race conditions to rely on file system - * event timestamps.) - * - * The is a simple integer incremented whenever the - * daemon needs to make its state public. For example, if 1000 file - * system events come in, but no clients have requested the data, - * the daemon can continue to accumulate file changes in the same - * bin and does not need to advance the sequence number. However, - * as soon as a client does arrive, the daemon needs to start a new - * bin and increment the sequence number. - * - * The sequence number serves as the boundary between 2 sets - * of bins -- the older ones that the client has already seen - * and the newer ones that it hasn't. - * - * When a new is created, the is reset to - * zero. - * - * - * About Token Ids - * =============== - * - * A new token_id is created: - * - * [1] each time the daemon is started. - * - * [2] any time that the daemon must re-sync with the filesystem - * (such as when the kernel drops or we miss events on a very - * active volume). - * - * [3] in response to a client "flush" command (for dropped event - * testing). - * - * When a new token_id is created, the daemon is free to discard all - * cached filesystem events associated with any previous token_ids. - * Events associated with a non-current token_id will never be sent - * to a client. A token_id change implicitly means that the daemon - * has gap in its event history. - * - * Therefore, clients that present a token with a stale (non-current) - * token_id will always be given a trivial response. - */ -struct fsmonitor_token_data { - struct strbuf token_id; - struct fsmonitor_batch *batch_head; - struct fsmonitor_batch *batch_tail; - uint64_t client_ref_count; -}; - -static struct fsmonitor_token_data *fsmonitor_new_token_data(void) -{ - static int test_env_value = -1; - static uint64_t flush_count = 0; - struct fsmonitor_token_data *token; - - CALLOC_ARRAY(token, 1); - - strbuf_init(&token->token_id, 0); - token->batch_head = NULL; - token->batch_tail = NULL; - token->client_ref_count = 0; - - if (test_env_value < 0) - test_env_value = git_env_bool("GIT_TEST_FSMONITOR_TOKEN", 0); - - if (!test_env_value) { - struct timeval tv; - struct tm tm; - time_t secs; - - gettimeofday(&tv, NULL); - secs = tv.tv_sec; - gmtime_r(&secs, &tm); - - strbuf_addf(&token->token_id, - "%"PRIu64".%d.%4d%02d%02dT%02d%02d%02d.%06ldZ", - flush_count++, - getpid(), - tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, - tm.tm_hour, tm.tm_min, tm.tm_sec, - (long)tv.tv_usec); - } else { - strbuf_addf(&token->token_id, "test_%08x", test_env_value++); - } - - return token; -} - static ipc_server_application_cb handle_client; static int handle_client(void *data, @@ -397,7 +283,7 @@ static int fsmonitor_run_daemon(void) pthread_mutex_init(&state.main_lock, NULL); state.error_code = 0; - state.current_token_data = fsmonitor_new_token_data(); + state.current_token_data = NULL; /* Prepare to (recursively) watch the directory. */ strbuf_init(&state.path_worktree_watch, 0); From d27ea8bbabd57581d9ebdf1225468b74882521de Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:18 -0400 Subject: [PATCH 031/110] fixup! fsmonitor--daemon: add pathname classification This reverts commit 89265708a0b96f5a35a3c96be4ae3a04492ee9f9. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 81 ------------------------------------- fsmonitor--daemon.h | 61 ---------------------------- 2 files changed, 142 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 2f89452d93b55d..5483549d0df1bf 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -121,87 +121,6 @@ static int handle_client(void *data, return result; } -#define FSMONITOR_COOKIE_PREFIX ".fsmonitor-daemon-" - -enum fsmonitor_path_type fsmonitor_classify_path_workdir_relative( - const char *rel) -{ - if (fspathncmp(rel, ".git", 4)) - return IS_WORKDIR_PATH; - rel += 4; - - if (!*rel) - return IS_DOT_GIT; - if (*rel != '/') - return IS_WORKDIR_PATH; /* e.g. .gitignore */ - rel++; - - if (!fspathncmp(rel, FSMONITOR_COOKIE_PREFIX, - strlen(FSMONITOR_COOKIE_PREFIX))) - return IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX; - - return IS_INSIDE_DOT_GIT; -} - -enum fsmonitor_path_type fsmonitor_classify_path_gitdir_relative( - const char *rel) -{ - if (!fspathncmp(rel, FSMONITOR_COOKIE_PREFIX, - strlen(FSMONITOR_COOKIE_PREFIX))) - return IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX; - - return IS_INSIDE_GITDIR; -} - -static enum fsmonitor_path_type try_classify_workdir_abs_path( - struct fsmonitor_daemon_state *state, - const char *path) -{ - const char *rel; - - if (fspathncmp(path, state->path_worktree_watch.buf, - state->path_worktree_watch.len)) - return IS_OUTSIDE_CONE; - - rel = path + state->path_worktree_watch.len; - - if (!*rel) - return IS_WORKDIR_PATH; /* it is the root dir exactly */ - if (*rel != '/') - return IS_OUTSIDE_CONE; - rel++; - - return fsmonitor_classify_path_workdir_relative(rel); -} - -enum fsmonitor_path_type fsmonitor_classify_path_absolute( - struct fsmonitor_daemon_state *state, - const char *path) -{ - const char *rel; - enum fsmonitor_path_type t; - - t = try_classify_workdir_abs_path(state, path); - if (state->nr_paths_watching == 1) - return t; - if (t != IS_OUTSIDE_CONE) - return t; - - if (fspathncmp(path, state->path_gitdir_watch.buf, - state->path_gitdir_watch.len)) - return IS_OUTSIDE_CONE; - - rel = path + state->path_gitdir_watch.len; - - if (!*rel) - return IS_GITDIR; /* it is the exactly */ - if (*rel != '/') - return IS_OUTSIDE_CONE; - rel++; - - return fsmonitor_classify_path_gitdir_relative(rel); -} - static void *fsm_listen__thread_proc(void *_state) { struct fsmonitor_daemon_state *state = _state; diff --git a/fsmonitor--daemon.h b/fsmonitor--daemon.h index 7bbb3a27a1ce1d..3009c1a83de7f6 100644 --- a/fsmonitor--daemon.h +++ b/fsmonitor--daemon.h @@ -30,66 +30,5 @@ struct fsmonitor_daemon_state { struct ipc_server_data *ipc_server_data; }; -/* - * Pathname classifications. - * - * The daemon classifies the pathnames that it receives from file - * system notification events into the following categories and uses - * that to decide whether clients are told about them. (And to watch - * for file system synchronization events.) - * - * The client should only care about paths within the working - * directory proper (inside the working directory and not ".git" nor - * inside of ".git/"). That is, the client has read the index and is - * asking for a list of any paths in the working directory that have - * been modified since the last token. The client does not care about - * file system changes within the .git directory (such as new loose - * objects or packfiles). So the client will only receive paths that - * are classified as IS_WORKDIR_PATH. - * - * The daemon uses the IS_DOT_GIT and IS_GITDIR internally to mean the - * exact ".git" directory or GITDIR. If the daemon receives a delete - * event for either of these directories, it will automatically - * shutdown, for example. - * - * Note that the daemon DOES NOT explicitly watch nor special case the - * ".git/index" file. The daemon does not read the index and does not - * have any internal index-relative state. The daemon only collects - * the set of modified paths within the working directory. - */ -enum fsmonitor_path_type { - IS_WORKDIR_PATH = 0, - - IS_DOT_GIT, - IS_INSIDE_DOT_GIT, - IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX, - - IS_GITDIR, - IS_INSIDE_GITDIR, - IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX, - - IS_OUTSIDE_CONE, -}; - -/* - * Classify a pathname relative to the root of the working directory. - */ -enum fsmonitor_path_type fsmonitor_classify_path_workdir_relative( - const char *relative_path); - -/* - * Classify a pathname relative to a that is external to the - * worktree directory. - */ -enum fsmonitor_path_type fsmonitor_classify_path_gitdir_relative( - const char *relative_path); - -/* - * Classify an absolute pathname received from a filesystem event. - */ -enum fsmonitor_path_type fsmonitor_classify_path_absolute( - struct fsmonitor_daemon_state *state, - const char *path); - #endif /* HAVE_FSMONITOR_DAEMON_BACKEND */ #endif /* FSMONITOR_DAEMON_H */ From eee16aa4aaa132893d24b033901a7063e167820e Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:20 -0400 Subject: [PATCH 032/110] fixup! fsmonitor--daemon: do not try to operate on bare repos This reverts commit eab07e13cc2d91931102e5ba3ffc14f63bf7a5ff. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 11 ----- builtin/update-index.c | 8 ---- fsmonitor-settings.c | 83 ++++--------------------------------- fsmonitor-settings.h | 11 ----- t/t7519-status-fsmonitor.sh | 26 ------------ 5 files changed, 9 insertions(+), 130 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 5483549d0df1bf..268adbd5de855a 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -490,17 +490,6 @@ int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) die(_("invalid 'ipc-threads' value (%d)"), fsmonitor__ipc_threads); - prepare_repo_settings(the_repository); - - fsm_settings__set_ipc(the_repository); - if (fsm_settings__get_mode(the_repository) == FSMONITOR_MODE_INCOMPATIBLE) { - struct strbuf buf_reason = STRBUF_INIT; - fsm_settings__get_reason(the_repository, &buf_reason); - error("%s '%s'", buf_reason.buf, xgetcwd()); - strbuf_release(&buf_reason); - return -1; - } - if (!strcmp(subcmd, "start")) return !!try_to_start_background_daemon(); diff --git a/builtin/update-index.c b/builtin/update-index.c index 38bf0ff4bd5a21..25dae7360a601f 100644 --- a/builtin/update-index.c +++ b/builtin/update-index.c @@ -1218,14 +1218,6 @@ int cmd_update_index(int argc, const char **argv, const char *prefix) if (fsmonitor > 0) { enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); - if (fsm_mode == FSMONITOR_MODE_INCOMPATIBLE) { - struct strbuf buf_reason = STRBUF_INIT; - fsm_settings__get_reason(r, &buf_reason); - error("%s", buf_reason.buf); - strbuf_release(&buf_reason); - return -1; - } - if (fsm_mode == FSMONITOR_MODE_DISABLED) { warning(_("core.useBuiltinFSMonitor is unset; " "set it if you really want to enable the " diff --git a/fsmonitor-settings.c b/fsmonitor-settings.c index 5a803a41d57434..2770266f5ee1ad 100644 --- a/fsmonitor-settings.c +++ b/fsmonitor-settings.c @@ -9,57 +9,19 @@ */ struct fsmonitor_settings { enum fsmonitor_mode mode; - enum fsmonitor_reason reason; char *hook_path; }; -static void set_incompatible(struct repository *r, - enum fsmonitor_reason reason) -{ - struct fsmonitor_settings *s = r->settings.fsmonitor; - - s->mode = FSMONITOR_MODE_INCOMPATIBLE; - s->reason = reason; -} - -static int check_for_incompatible(struct repository *r) -{ - if (!r->worktree) { - /* - * Bare repositories don't have a working directory and - * therefore have nothing to watch. - */ - set_incompatible(r, FSMONITOR_REASON_BARE); - return 1; - } - - return 0; -} - -static struct fsmonitor_settings *s_init(struct repository *r) -{ - if (!r->settings.fsmonitor) - CALLOC_ARRAY(r->settings.fsmonitor, 1); - - return r->settings.fsmonitor; -} - void fsm_settings__set_ipc(struct repository *r) { - struct fsmonitor_settings *s = s_init(r); - - if (check_for_incompatible(r)) - return; + struct fsmonitor_settings *s = r->settings.fsmonitor; s->mode = FSMONITOR_MODE_IPC; } void fsm_settings__set_hook(struct repository *r, const char *path) { - struct fsmonitor_settings *s = s_init(r); - - if (check_for_incompatible(r)) - return; + struct fsmonitor_settings *s = r->settings.fsmonitor; s->mode = FSMONITOR_MODE_HOOK; s->hook_path = strdup(path); @@ -67,10 +29,9 @@ void fsm_settings__set_hook(struct repository *r, const char *path) void fsm_settings__set_disabled(struct repository *r) { - struct fsmonitor_settings *s = s_init(r); + struct fsmonitor_settings *s = r->settings.fsmonitor; s->mode = FSMONITOR_MODE_DISABLED; - s->reason = FSMONITOR_REASON_ZERO; FREE_AND_NULL(s->hook_path); } @@ -104,6 +65,12 @@ static int check_for_hook(struct repository *r) static void lookup_fsmonitor_settings(struct repository *r) { + struct fsmonitor_settings *s; + + CALLOC_ARRAY(s, 1); + + r->settings.fsmonitor = s; + if (check_for_ipc(r)) return; @@ -128,35 +95,3 @@ const char *fsm_settings__get_hook_path(struct repository *r) return r->settings.fsmonitor->hook_path; } - -static void create_reason_message(struct repository *r, - struct strbuf *buf_reason) -{ - struct fsmonitor_settings *s = r->settings.fsmonitor; - - switch (s->reason) { - case FSMONITOR_REASON_ZERO: - return; - - case FSMONITOR_REASON_BARE: - strbuf_addstr(buf_reason, - _("bare repos are incompatible with fsmonitor")); - return; - - default: - BUG("Unhandled case in create_reason_message '%d'", s->reason); - } -} -enum fsmonitor_reason fsm_settings__get_reason(struct repository *r, - struct strbuf *buf_reason) -{ - strbuf_reset(buf_reason); - - if (!r->settings.fsmonitor) - lookup_fsmonitor_settings(r); - - if (r->settings.fsmonitor->mode == FSMONITOR_MODE_INCOMPATIBLE) - create_reason_message(r, buf_reason); - - return r->settings.fsmonitor->reason; -} diff --git a/fsmonitor-settings.h b/fsmonitor-settings.h index eb45524b1f2297..50b2923461656c 100644 --- a/fsmonitor-settings.h +++ b/fsmonitor-settings.h @@ -4,28 +4,17 @@ struct repository; enum fsmonitor_mode { - FSMONITOR_MODE_INCOMPATIBLE = -1, /* see _reason */ FSMONITOR_MODE_DISABLED = 0, FSMONITOR_MODE_HOOK = 1, /* core.fsmonitor */ FSMONITOR_MODE_IPC = 2, /* core.useBuiltinFSMonitor */ }; -/* - * Incompatibility reasons. - */ -enum fsmonitor_reason { - FSMONITOR_REASON_ZERO = 0, - FSMONITOR_REASON_BARE = 1, -}; - void fsm_settings__set_ipc(struct repository *r); void fsm_settings__set_hook(struct repository *r, const char *path); void fsm_settings__set_disabled(struct repository *r); enum fsmonitor_mode fsm_settings__get_mode(struct repository *r); const char *fsm_settings__get_hook_path(struct repository *r); -enum fsmonitor_reason fsm_settings__get_reason(struct repository *r, - struct strbuf *buf_reason); struct fsmonitor_settings; diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index b9407d4738b129..deea88d4431d23 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -55,32 +55,6 @@ test_lazy_prereq UNTRACKED_CACHE ' test $ret -ne 1 ' -# Test that we detect and disallow repos that are incompatible with FSMonitor. -test_expect_success 'incompatible bare repo' ' - test_when_finished "rm -rf ./bare-clone actual expect" && - git init --bare bare-clone && - cat >expect <<-\EOF && - error: bare repos are incompatible with fsmonitor - EOF - - test_must_fail \ - git -C ./bare-clone -c core.fsmonitor=foo \ - update-index --fsmonitor 2>actual && - test_cmp expect actual && - - test_must_fail \ - git -C ./bare-clone -c core.usebuiltinfsmonitor=true \ - update-index --fsmonitor 2>actual && - test_cmp expect actual -' - -test_expect_success FSMONITOR_DAEMON 'run fsmonitor-daemon in bare repo' ' - test_when_finished "rm -rf ./bare-clone actual" && - git init --bare bare-clone && - test_must_fail git -C ./bare-clone fsmonitor--daemon run 2>actual && - grep "bare repos are incompatible with fsmonitor" actual -' - test_expect_success 'setup' ' mkdir -p .git/hooks && : >tracked && From 4ef31733492da8b7b9dcdb0bae58ba709f20757b Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:22 -0400 Subject: [PATCH 033/110] fixup! fsmonitor--daemon: implement 'start' command This reverts commit f5c5085496b1977588a8855ec694968e3066cf1a. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 208 ------------------------------------ 1 file changed, 208 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 268adbd5de855a..5e2749747a9658 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -9,7 +9,6 @@ #include "khash.h" static const char * const builtin_fsmonitor__daemon_usage[] = { - N_("git fsmonitor--daemon start []"), N_("git fsmonitor--daemon run []"), N_("git fsmonitor--daemon stop"), N_("git fsmonitor--daemon status"), @@ -23,9 +22,6 @@ static const char * const builtin_fsmonitor__daemon_usage[] = { #define FSMONITOR__IPC_THREADS "fsmonitor.ipcthreads" static int fsmonitor__ipc_threads = 8; -#define FSMONITOR__START_TIMEOUT "fsmonitor.starttimeout" -static int fsmonitor__start_timeout_sec = 60; - static int fsmonitor_config(const char *var, const char *value, void *cb) { if (!strcmp(var, FSMONITOR__IPC_THREADS)) { @@ -37,15 +33,6 @@ static int fsmonitor_config(const char *var, const char *value, void *cb) return 0; } - if (!strcmp(var, FSMONITOR__START_TIMEOUT)) { - int i = git_config_int(var, value); - if (i < 0) - return error(_("value of '%s' out of range: %d"), - FSMONITOR__START_TIMEOUT, i); - fsmonitor__start_timeout_sec = i; - return 0; - } - return git_default_config(var, value, cb); } @@ -269,194 +256,6 @@ static int try_to_run_foreground_daemon(void) return !!fsmonitor_run_daemon(); } -#ifdef GIT_WINDOWS_NATIVE -/* - * Create a background process to run the daemon. It should be completely - * disassociated from the terminal. - * - * Conceptually like `daemonize()` but different because Windows does not - * have `fork(2)`. Spawn a normal Windows child process but without the - * limitations of `start_command()` and `finish_command()`. - * - * The child process execs the "git fsmonitor--daemon run" command. - * - * The current process returns so that the caller can wait for the child - * to startup before exiting. - */ -static int spawn_fsmonitor(pid_t *pid) -{ - char git_exe[MAX_PATH]; - struct strvec args = STRVEC_INIT; - int in, out; - - GetModuleFileNameA(NULL, git_exe, MAX_PATH); - - in = open("/dev/null", O_RDONLY); - out = open("/dev/null", O_WRONLY); - - strvec_push(&args, git_exe); - strvec_push(&args, "fsmonitor--daemon"); - strvec_push(&args, "run"); - strvec_pushf(&args, "--ipc-threads=%d", fsmonitor__ipc_threads); - - *pid = mingw_spawnvpe(args.v[0], args.v, NULL, NULL, in, out, out); - close(in); - close(out); - - strvec_clear(&args); - - if (*pid < 0) - return error(_("could not spawn fsmonitor--daemon in the background")); - - return 0; -} -#else -/* - * Create a background process to run the daemon. It should be completely - * disassociated from the terminal. - * - * This is adapted from `daemonize()`. Use `fork()` to directly - * create and run the daemon in the child process. - * - * The fork-child can just call the run code; it does not need to exec - * it. - * - * The fork-parent returns the child PID so that we can wait for the - * child to startup before exiting. - */ -static int spawn_fsmonitor(pid_t *pid) -{ - *pid = fork(); - - switch (*pid) { - case 0: - if (setsid() == -1) - error_errno(_("setsid failed")); - close(0); - close(1); - close(2); - sanitize_stdfds(); - - return !!fsmonitor_run_daemon(); - - case -1: - return error_errno(_("could not spawn fsmonitor--daemon in the background")); - - default: - return 0; - } -} -#endif - -/* - * This is adapted from `wait_or_whine()`. Watch the child process and - * let it get started and begin listening for requests on the socket - * before reporting our success. - */ -static int wait_for_startup(pid_t pid_child) -{ - int status; - pid_t pid_seen; - enum ipc_active_state s; - time_t time_limit, now; - - time(&time_limit); - time_limit += fsmonitor__start_timeout_sec; - - for (;;) { - pid_seen = waitpid(pid_child, &status, WNOHANG); - - if (pid_seen == -1) - return error_errno(_("waitpid failed")); - else if (pid_seen == 0) { - /* - * The child is still running (this should be - * the normal case). Try to connect to it on - * the socket and see if it is ready for - * business. - * - * If there is another daemon already running, - * our child will fail to start (possibly - * after a timeout on the lock), but we don't - * care (who responds) if the socket is live. - */ - s = fsmonitor_ipc__get_state(); - if (s == IPC_STATE__LISTENING) - return 0; - - time(&now); - if (now > time_limit) - return error(_("fsmonitor--daemon not online yet")); - } else if (pid_seen == pid_child) { - /* - * The new child daemon process shutdown while - * it was starting up, so it is not listening - * on the socket. - * - * Try to ping the socket in the odd chance - * that another daemon started (or was already - * running) while our child was starting. - * - * Again, we don't care who services the socket. - */ - s = fsmonitor_ipc__get_state(); - if (s == IPC_STATE__LISTENING) - return 0; - - /* - * We don't care about the WEXITSTATUS() nor - * any of the WIF*(status) values because - * `cmd_fsmonitor__daemon()` does the `!!result` - * trick on all function return values. - * - * So it is sufficient to just report the - * early shutdown as an error. - */ - return error(_("fsmonitor--daemon failed to start")); - } else - return error(_("waitpid is confused")); - } -} - -static int try_to_start_background_daemon(void) -{ - pid_t pid_child; - int ret; - - /* - * Before we try to create a background daemon process, see - * if a daemon process is already listening. This makes it - * easier for us to report an already-listening error to the - * console, since our spawn/daemon can only report the success - * of creating the background process (and not whether it - * immediately exited). - */ - if (fsmonitor_ipc__get_state() == IPC_STATE__LISTENING) - die("fsmonitor--daemon is already running '%s'", - the_repository->worktree); - - printf(_("starting fsmonitor-daemon in '%s'\n"), - the_repository->worktree); - fflush(stdout); - - /* - * Run the actual daemon in a background process. - */ - ret = spawn_fsmonitor(&pid_child); - if (pid_child <= 0) - return ret; - - /* - * Wait (with timeout) for the background child process get - * started and begin listening on the socket/pipe. This makes - * the "start" command more synchronous and more reliable in - * tests. - */ - ret = wait_for_startup(pid_child); - - return ret; -} - int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) { const char *subcmd; @@ -465,10 +264,6 @@ int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) OPT_INTEGER(0, "ipc-threads", &fsmonitor__ipc_threads, N_("use ipc worker threads")), - OPT_INTEGER(0, "start-timeout", - &fsmonitor__start_timeout_sec, - N_("Max seconds to wait for background daemon startup")), - OPT_END() }; @@ -490,9 +285,6 @@ int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) die(_("invalid 'ipc-threads' value (%d)"), fsmonitor__ipc_threads); - if (!strcmp(subcmd, "start")) - return !!try_to_start_background_daemon(); - if (!strcmp(subcmd, "run")) return !!try_to_run_foreground_daemon(); From d5fd4ed9e881d2b899a853e79028928af9afef36 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:25 -0400 Subject: [PATCH 034/110] fixup! fsmonitor--daemon: implement 'run' command This reverts commit 7d39bbb344bec9efb7f80da1bd6f77c2947a4ad9. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 210 +----------------------------------- fsmonitor--daemon.h | 34 ------ 2 files changed, 1 insertion(+), 243 deletions(-) delete mode 100644 fsmonitor--daemon.h diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 5e2749747a9658..62efd5ea787afe 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -3,39 +3,16 @@ #include "parse-options.h" #include "fsmonitor.h" #include "fsmonitor-ipc.h" -#include "compat/fsmonitor/fsm-listen.h" -#include "fsmonitor--daemon.h" #include "simple-ipc.h" #include "khash.h" static const char * const builtin_fsmonitor__daemon_usage[] = { - N_("git fsmonitor--daemon run []"), N_("git fsmonitor--daemon stop"), N_("git fsmonitor--daemon status"), NULL }; #ifdef HAVE_FSMONITOR_DAEMON_BACKEND -/* - * Global state loaded from config. - */ -#define FSMONITOR__IPC_THREADS "fsmonitor.ipcthreads" -static int fsmonitor__ipc_threads = 8; - -static int fsmonitor_config(const char *var, const char *value, void *cb) -{ - if (!strcmp(var, FSMONITOR__IPC_THREADS)) { - int i = git_config_int(var, value); - if (i < 1) - return error(_("value of '%s' out of range: %d"), - FSMONITOR__IPC_THREADS, i); - fsmonitor__ipc_threads = i; - return 0; - } - - return git_default_config(var, value, cb); -} - /* * Acting as a CLIENT. * @@ -80,190 +57,11 @@ static int do_as_client__status(void) } } -static ipc_server_application_cb handle_client; - -static int handle_client(void *data, - const char *command, size_t command_len, - ipc_server_reply_cb *reply, - struct ipc_server_reply_data *reply_data) -{ - /* struct fsmonitor_daemon_state *state = data; */ - int result; - - /* - * The Simple IPC API now supports {char*, len} arguments, but - * FSMonitor always uses proper null-terminated strings, so - * we can ignore the command_len argument. (Trust, but verify.) - */ - if (command_len != strlen(command)) - BUG("FSMonitor assumes text messages"); - - trace2_region_enter("fsmonitor", "handle_client", the_repository); - trace2_data_string("fsmonitor", the_repository, "request", command); - - result = 0; /* TODO Do something here. */ - - trace2_region_leave("fsmonitor", "handle_client", the_repository); - - return result; -} - -static void *fsm_listen__thread_proc(void *_state) -{ - struct fsmonitor_daemon_state *state = _state; - - trace2_thread_start("fsm-listen"); - - trace_printf_key(&trace_fsmonitor, "Watching: worktree '%s'", - state->path_worktree_watch.buf); - if (state->nr_paths_watching > 1) - trace_printf_key(&trace_fsmonitor, "Watching: gitdir '%s'", - state->path_gitdir_watch.buf); - - fsm_listen__loop(state); - - trace2_thread_exit(); - return NULL; -} - -static int fsmonitor_run_daemon_1(struct fsmonitor_daemon_state *state) -{ - struct ipc_server_opts ipc_opts = { - .nr_threads = fsmonitor__ipc_threads, - - /* - * We know that there are no other active threads yet, - * so we can let the IPC layer temporarily chdir() if - * it needs to when creating the server side of the - * Unix domain socket. - */ - .uds_disallow_chdir = 0 - }; - - /* - * Start the IPC thread pool before the we've started the file - * system event listener thread so that we have the IPC handle - * before we need it. - */ - if (ipc_server_run_async(&state->ipc_server_data, - fsmonitor_ipc__get_path(), &ipc_opts, - handle_client, state)) - return error(_("could not start IPC thread pool")); - - /* - * Start the fsmonitor listener thread to collect filesystem - * events. - */ - if (pthread_create(&state->listener_thread, NULL, - fsm_listen__thread_proc, state) < 0) { - ipc_server_stop_async(state->ipc_server_data); - ipc_server_await(state->ipc_server_data); - - return error(_("could not start fsmonitor listener thread")); - } - - /* - * The daemon is now fully functional in background threads. - * Wait for the IPC thread pool to shutdown (whether by client - * request or from filesystem activity). - */ - ipc_server_await(state->ipc_server_data); - - /* - * The fsmonitor listener thread may have received a shutdown - * event from the IPC thread pool, but it doesn't hurt to tell - * it again. And wait for it to shutdown. - */ - fsm_listen__stop_async(state); - pthread_join(state->listener_thread, NULL); - - return state->error_code; -} - -static int fsmonitor_run_daemon(void) -{ - struct fsmonitor_daemon_state state; - int err; - - memset(&state, 0, sizeof(state)); - - pthread_mutex_init(&state.main_lock, NULL); - state.error_code = 0; - state.current_token_data = NULL; - - /* Prepare to (recursively) watch the directory. */ - strbuf_init(&state.path_worktree_watch, 0); - strbuf_addstr(&state.path_worktree_watch, absolute_path(get_git_work_tree())); - state.nr_paths_watching = 1; - - /* - * We create and delete cookie files somewhere inside the .git - * directory to help us keep sync with the file system. If - * ".git" is not a directory, then is not inside the - * cone of , so set up a second watch to watch - * the so that we get events for the cookie files. - */ - strbuf_init(&state.path_gitdir_watch, 0); - strbuf_addbuf(&state.path_gitdir_watch, &state.path_worktree_watch); - strbuf_addstr(&state.path_gitdir_watch, "/.git"); - if (!is_directory(state.path_gitdir_watch.buf)) { - strbuf_reset(&state.path_gitdir_watch); - strbuf_addstr(&state.path_gitdir_watch, absolute_path(get_git_dir())); - state.nr_paths_watching = 2; - } - - /* - * Confirm that we can create platform-specific resources for the - * filesystem listener before we bother starting all the threads. - */ - if (fsm_listen__ctor(&state)) { - err = error(_("could not initialize listener thread")); - goto done; - } - - err = fsmonitor_run_daemon_1(&state); - -done: - pthread_mutex_destroy(&state.main_lock); - fsm_listen__dtor(&state); - - ipc_server_free(state.ipc_server_data); - - strbuf_release(&state.path_worktree_watch); - strbuf_release(&state.path_gitdir_watch); - - return err; -} - -static int try_to_run_foreground_daemon(void) -{ - /* - * Technically, we don't need to probe for an existing daemon - * process, since we could just call `fsmonitor_run_daemon()` - * and let it fail if the pipe/socket is busy. - * - * However, this method gives us a nicer error message for a - * common error case. - */ - if (fsmonitor_ipc__get_state() == IPC_STATE__LISTENING) - die("fsmonitor--daemon is already running '%s'", - the_repository->worktree); - - printf(_("running fsmonitor-daemon in '%s'\n"), - the_repository->worktree); - fflush(stdout); - - return !!fsmonitor_run_daemon(); -} - int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) { const char *subcmd; struct option options[] = { - OPT_INTEGER(0, "ipc-threads", - &fsmonitor__ipc_threads, - N_("use ipc worker threads")), OPT_END() }; @@ -273,7 +71,7 @@ int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) if (argc == 2 && !strcmp(argv[1], "-h")) usage_with_options(builtin_fsmonitor__daemon_usage, options); - git_config(fsmonitor_config, NULL); + git_config(git_default_config, NULL); subcmd = argv[1]; argv--; @@ -281,12 +79,6 @@ int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) argc = parse_options(argc, argv, prefix, options, builtin_fsmonitor__daemon_usage, 0); - if (fsmonitor__ipc_threads < 1) - die(_("invalid 'ipc-threads' value (%d)"), - fsmonitor__ipc_threads); - - if (!strcmp(subcmd, "run")) - return !!try_to_run_foreground_daemon(); if (!strcmp(subcmd, "stop")) return !!do_as_client__send_stop(); diff --git a/fsmonitor--daemon.h b/fsmonitor--daemon.h deleted file mode 100644 index 3009c1a83de7f6..00000000000000 --- a/fsmonitor--daemon.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef FSMONITOR_DAEMON_H -#define FSMONITOR_DAEMON_H - -#ifdef HAVE_FSMONITOR_DAEMON_BACKEND - -#include "cache.h" -#include "dir.h" -#include "run-command.h" -#include "simple-ipc.h" -#include "thread-utils.h" - -struct fsmonitor_batch; -struct fsmonitor_token_data; - -struct fsmonitor_daemon_backend_data; /* opaque platform-specific data */ - -struct fsmonitor_daemon_state { - pthread_t listener_thread; - pthread_mutex_t main_lock; - - struct strbuf path_worktree_watch; - struct strbuf path_gitdir_watch; - int nr_paths_watching; - - struct fsmonitor_token_data *current_token_data; - - int error_code; - struct fsmonitor_daemon_backend_data *backend_data; - - struct ipc_server_data *ipc_server_data; -}; - -#endif /* HAVE_FSMONITOR_DAEMON_BACKEND */ -#endif /* FSMONITOR_DAEMON_H */ From d953cad8a9da2a60c24768617fab826c4839487c Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:27 -0400 Subject: [PATCH 035/110] fixup! fsm-listen-darwin: stub in backend for Darwin This reverts commit a6784cb37843d6d669fc5dd1cf8778d4df041a86. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-listen-darwin.c | 20 -------------------- config.mak.uname | 2 -- contrib/buildsystems/CMakeLists.txt | 3 --- 3 files changed, 25 deletions(-) delete mode 100644 compat/fsmonitor/fsm-listen-darwin.c diff --git a/compat/fsmonitor/fsm-listen-darwin.c b/compat/fsmonitor/fsm-listen-darwin.c deleted file mode 100644 index c84e3344ab99b4..00000000000000 --- a/compat/fsmonitor/fsm-listen-darwin.c +++ /dev/null @@ -1,20 +0,0 @@ -#include "cache.h" -#include "fsmonitor.h" -#include "fsm-listen.h" - -int fsm_listen__ctor(struct fsmonitor_daemon_state *state) -{ - return -1; -} - -void fsm_listen__dtor(struct fsmonitor_daemon_state *state) -{ -} - -void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) -{ -} - -void fsm_listen__loop(struct fsmonitor_daemon_state *state) -{ -} diff --git a/config.mak.uname b/config.mak.uname index a27ccb583d43f2..88237b8f19d0f4 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -147,8 +147,6 @@ ifeq ($(uname_S),Darwin) MSGFMT = /usr/local/opt/gettext/bin/msgfmt endif endif - FSMONITOR_DAEMON_BACKEND = darwin - BASIC_LDFLAGS += -framework CoreServices endif ifeq ($(uname_S),SunOS) NEEDS_SOCKET = YesPlease diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 79e4b0dae384e4..041c3ababcb2d6 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -295,9 +295,6 @@ endif() if(CMAKE_SYSTEM_NAME STREQUAL "Windows") add_compile_definitions(HAVE_FSMONITOR_DAEMON_BACKEND) list(APPEND compat_SOURCES compat/fsmonitor/fsm-listen-win32.c) -elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") - add_compile_definitions(HAVE_FSMONITOR_DAEMON_BACKEND) - list(APPEND compat_SOURCES compat/fsmonitor/fsm-listen-darwin.c) endif() set(EXE_EXTENSION ${CMAKE_EXECUTABLE_SUFFIX}) From c1b76c04bccc29ca7a3d185f14c3ccc2d7bf985c Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:30 -0400 Subject: [PATCH 036/110] fixup! fsm-listen-win32: stub in backend for Windows This reverts commit 1b42506abac09d3d7b77f027a81fcc67ccf41820. Signed-off-by: Jeff Hostetler --- Makefile | 13 -------- compat/fsmonitor/fsm-listen-win32.c | 21 ------------- compat/fsmonitor/fsm-listen.h | 49 ----------------------------- config.mak.uname | 2 -- contrib/buildsystems/CMakeLists.txt | 5 --- repo-settings.c | 1 - 6 files changed, 91 deletions(-) delete mode 100644 compat/fsmonitor/fsm-listen-win32.c delete mode 100644 compat/fsmonitor/fsm-listen.h diff --git a/Makefile b/Makefile index dca63dd2d55cc0..4a682550d4c03f 100644 --- a/Makefile +++ b/Makefile @@ -471,11 +471,6 @@ all:: # directory, and the JSON compilation database 'compile_commands.json' will be # created at the root of the repository. # -# If your platform supports a built-in fsmonitor backend, set -# FSMONITOR_DAEMON_BACKEND to the "" of the corresponding -# `compat/fsmonitor/fsm-listen-.c` that implements the -# `fsm_listen__*()` routines. -# # Define DEVELOPER to enable more compiler warnings. Compiler version # and family are auto detected, but could be overridden by defining # COMPILER_FEATURES (see config.mak.dev). You can still set @@ -1946,11 +1941,6 @@ ifdef NEED_ACCESS_ROOT_HANDLER COMPAT_OBJS += compat/access.o endif -ifdef FSMONITOR_DAEMON_BACKEND - COMPAT_CFLAGS += -DHAVE_FSMONITOR_DAEMON_BACKEND - COMPAT_OBJS += compat/fsmonitor/fsm-listen-$(FSMONITOR_DAEMON_BACKEND).o -endif - ifeq ($(TCLTK_PATH),) NO_TCLTK = NoThanks endif @@ -2832,9 +2822,6 @@ GIT-BUILD-OPTIONS: FORCE @echo PAGER_ENV=\''$(subst ','\'',$(subst ','\'',$(PAGER_ENV)))'\' >>$@+ @echo DC_SHA1=\''$(subst ','\'',$(subst ','\'',$(DC_SHA1)))'\' >>$@+ @echo X=\'$(X)\' >>$@+ -ifdef FSMONITOR_DAEMON_BACKEND - @echo FSMONITOR_DAEMON_BACKEND=\''$(subst ','\'',$(subst ','\'',$(FSMONITOR_DAEMON_BACKEND)))'\' >>$@+ -endif ifdef TEST_OUTPUT_DIRECTORY @echo TEST_OUTPUT_DIRECTORY=\''$(subst ','\'',$(subst ','\'',$(TEST_OUTPUT_DIRECTORY)))'\' >>$@+ endif diff --git a/compat/fsmonitor/fsm-listen-win32.c b/compat/fsmonitor/fsm-listen-win32.c deleted file mode 100644 index 916cbea254f8cf..00000000000000 --- a/compat/fsmonitor/fsm-listen-win32.c +++ /dev/null @@ -1,21 +0,0 @@ -#include "cache.h" -#include "config.h" -#include "fsmonitor.h" -#include "fsm-listen.h" - -void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) -{ -} - -void fsm_listen__loop(struct fsmonitor_daemon_state *state) -{ -} - -int fsm_listen__ctor(struct fsmonitor_daemon_state *state) -{ - return -1; -} - -void fsm_listen__dtor(struct fsmonitor_daemon_state *state) -{ -} diff --git a/compat/fsmonitor/fsm-listen.h b/compat/fsmonitor/fsm-listen.h deleted file mode 100644 index f0539349baf848..00000000000000 --- a/compat/fsmonitor/fsm-listen.h +++ /dev/null @@ -1,49 +0,0 @@ -#ifndef FSM_LISTEN_H -#define FSM_LISTEN_H - -/* This needs to be implemented by each backend */ - -#ifdef HAVE_FSMONITOR_DAEMON_BACKEND - -struct fsmonitor_daemon_state; - -/* - * Initialize platform-specific data for the fsmonitor listener thread. - * This will be called from the main thread PRIOR to staring the - * fsmonitor_fs_listener thread. - * - * Returns 0 if successful. - * Returns -1 otherwise. - */ -int fsm_listen__ctor(struct fsmonitor_daemon_state *state); - -/* - * Cleanup platform-specific data for the fsmonitor listener thread. - * This will be called from the main thread AFTER joining the listener. - */ -void fsm_listen__dtor(struct fsmonitor_daemon_state *state); - -/* - * The main body of the platform-specific event loop to watch for - * filesystem events. This will run in the fsmonitor_fs_listen thread. - * - * It should call `ipc_server_stop_async()` if the listener thread - * prematurely terminates (because of a filesystem error or if it - * detects that the .git directory has been deleted). (It should NOT - * do so if the listener thread receives a normal shutdown signal from - * the IPC layer.) - * - * It should set `state->error_code` to -1 if the daemon should exit - * with an error. - */ -void fsm_listen__loop(struct fsmonitor_daemon_state *state); - -/* - * Gently request that the fsmonitor listener thread shutdown. - * It does not wait for it to stop. The caller should do a JOIN - * to wait for it. - */ -void fsm_listen__stop_async(struct fsmonitor_daemon_state *state); - -#endif /* HAVE_FSMONITOR_DAEMON_BACKEND */ -#endif /* FSM_LISTEN_H */ diff --git a/config.mak.uname b/config.mak.uname index 88237b8f19d0f4..d92da1fff13781 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -420,7 +420,6 @@ ifeq ($(uname_S),Windows) # so we don't need this: # # SNPRINTF_RETURNS_BOGUS = YesPlease - FSMONITOR_DAEMON_BACKEND = win32 NO_SVN_TESTS = YesPlease RUNTIME_PREFIX = YesPlease HAVE_WPGMPTR = YesWeDo @@ -608,7 +607,6 @@ ifneq (,$(findstring MINGW,$(uname_S))) NO_STRTOUMAX = YesPlease NO_MKDTEMP = YesPlease NO_SVN_TESTS = YesPlease - FSMONITOR_DAEMON_BACKEND = win32 RUNTIME_PREFIX = YesPlease HAVE_WPGMPTR = YesWeDo NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 041c3ababcb2d6..a4ffbacfa4b83f 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -292,11 +292,6 @@ else() endif() endif() -if(CMAKE_SYSTEM_NAME STREQUAL "Windows") - add_compile_definitions(HAVE_FSMONITOR_DAEMON_BACKEND) - list(APPEND compat_SOURCES compat/fsmonitor/fsm-listen-win32.c) -endif() - set(EXE_EXTENSION ${CMAKE_EXECUTABLE_SUFFIX}) #header checks diff --git a/repo-settings.c b/repo-settings.c index 08c60a927473e2..e69dad1e776772 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -2,7 +2,6 @@ #include "config.h" #include "repository.h" #include "midx.h" -#include "compat/fsmonitor/fsm-listen.h" #define UPDATE_DEFAULT_BOOL(s,v) do { if (s == -1) { s = v; } } while(0) From 7fbbac3a2970d727a54151035413ffd63b072d42 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:32 -0400 Subject: [PATCH 037/110] fixup! t/helper/fsmonitor-client: create IPC client to talk to FSMonitor Daemon This reverts commit f6e27d05bc9074e6cce7c22006639614a37cd66b. Signed-off-by: Jeff Hostetler --- Makefile | 1 - t/helper/test-fsmonitor-client.c | 121 ------------------------------- t/helper/test-tool.c | 1 - t/helper/test-tool.h | 1 - 4 files changed, 124 deletions(-) delete mode 100644 t/helper/test-fsmonitor-client.c diff --git a/Makefile b/Makefile index 4a682550d4c03f..f62752c1cccdce 100644 --- a/Makefile +++ b/Makefile @@ -714,7 +714,6 @@ TEST_BUILTINS_OBJS += test-dump-split-index.o TEST_BUILTINS_OBJS += test-dump-untracked-cache.o TEST_BUILTINS_OBJS += test-example-decorate.o TEST_BUILTINS_OBJS += test-fast-rebase.o -TEST_BUILTINS_OBJS += test-fsmonitor-client.o TEST_BUILTINS_OBJS += test-genrandom.o TEST_BUILTINS_OBJS += test-genzeros.o TEST_BUILTINS_OBJS += test-getcwd.o diff --git a/t/helper/test-fsmonitor-client.c b/t/helper/test-fsmonitor-client.c deleted file mode 100644 index f7a5b3a32fab86..00000000000000 --- a/t/helper/test-fsmonitor-client.c +++ /dev/null @@ -1,121 +0,0 @@ -/* - * test-fsmonitor-client.c: client code to send commands/requests to - * a `git fsmonitor--daemon` daemon. - */ - -#include "test-tool.h" -#include "cache.h" -#include "parse-options.h" -#include "fsmonitor-ipc.h" - -#ifndef HAVE_FSMONITOR_DAEMON_BACKEND -int cmd__fsmonitor_client(int argc, const char **argv) -{ - die("fsmonitor--daemon not available on this platform"); -} -#else - -/* - * Read the `.git/index` to get the last token written to the - * FSMonitor Index Extension. - */ -static const char *get_token_from_index(void) -{ - struct index_state *istate = the_repository->index; - - if (do_read_index(istate, the_repository->index_file, 0) < 0) - die("unable to read index file"); - if (!istate->fsmonitor_last_update) - die("index file does not have fsmonitor extension"); - - return istate->fsmonitor_last_update; -} - -/* - * Send an IPC query to a `git-fsmonitor--daemon` daemon and - * ask for the changes since the given token or from the last - * token in the index extension. - * - * This will implicitly start a daemon process if necessary. The - * daemon process will persist after we exit. - */ -static int do_send_query(const char *token) -{ - struct strbuf answer = STRBUF_INIT; - int ret; - - if (!token || !*token) - token = get_token_from_index(); - - ret = fsmonitor_ipc__send_query(token, &answer); - if (ret < 0) - die(_("could not query fsmonitor--daemon")); - - write_in_full(1, answer.buf, answer.len); - strbuf_release(&answer); - - return 0; -} - -/* - * Send a "flush" command to the `git-fsmonitor--daemon` (if running) - * and tell it to flush its cache. - * - * This feature is primarily used by the test suite to simulate a loss of - * sync with the filesystem where we miss kernel events. - */ -static int do_send_flush(void) -{ - struct strbuf answer = STRBUF_INIT; - int ret; - - ret = fsmonitor_ipc__send_command("flush", &answer); - if (ret) - return ret; - - write_in_full(1, answer.buf, answer.len); - strbuf_release(&answer); - - return 0; -} - -int cmd__fsmonitor_client(int argc, const char **argv) -{ - const char *subcmd; - const char *token = NULL; - - const char * const fsmonitor_client_usage[] = { - N_("test-helper fsmonitor-client query []"), - N_("test-helper fsmonitor-client flush"), - NULL, - }; - - struct option options[] = { - OPT_STRING(0, "token", &token, N_("token"), - N_("command token to send to the server")), - OPT_END() - }; - - if (argc < 2) - usage_with_options(fsmonitor_client_usage, options); - - if (argc == 2 && !strcmp(argv[1], "-h")) - usage_with_options(fsmonitor_client_usage, options); - - subcmd = argv[1]; - argv--; - argc++; - - argc = parse_options(argc, argv, NULL, options, fsmonitor_client_usage, 0); - - setup_git_directory(); - - if (!strcmp(subcmd, "query")) - return !!do_send_query(token); - - if (!strcmp(subcmd, "flush")) - return !!do_send_flush(); - - die("Unhandled subcommand: '%s'", subcmd); -} -#endif diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index 7669b19177c0bc..5efa225cc32949 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -32,7 +32,6 @@ static struct test_cmd cmds[] = { { "dump-untracked-cache", cmd__dump_untracked_cache }, { "example-decorate", cmd__example_decorate }, { "fast-rebase", cmd__fast_rebase }, - { "fsmonitor-client", cmd__fsmonitor_client }, { "genrandom", cmd__genrandom }, { "genzeros", cmd__genzeros }, { "getcwd", cmd__getcwd }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index 52b008e020006f..b8712a5223958d 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -22,7 +22,6 @@ int cmd__dump_split_index(int argc, const char **argv); int cmd__dump_untracked_cache(int argc, const char **argv); int cmd__example_decorate(int argc, const char **argv); int cmd__fast_rebase(int argc, const char **argv); -int cmd__fsmonitor_client(int argc, const char **argv); int cmd__genrandom(int argc, const char **argv); int cmd__genzeros(int argc, const char **argv); int cmd__getcwd(int argc, const char **argv); From a9bfc01b1f71417e3e17a34bd2c762a88ba2a6d8 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:34 -0400 Subject: [PATCH 038/110] fixup! fsmonitor--daemon: implement 'stop' and 'status' commands This reverts commit 1866a62b4f5143b0e6e427c4594183a6276a33bf. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 51 ------------------------------------- 1 file changed, 51 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 62efd5ea787afe..df2bad531118c8 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -7,55 +7,10 @@ #include "khash.h" static const char * const builtin_fsmonitor__daemon_usage[] = { - N_("git fsmonitor--daemon stop"), - N_("git fsmonitor--daemon status"), NULL }; #ifdef HAVE_FSMONITOR_DAEMON_BACKEND -/* - * Acting as a CLIENT. - * - * Send a "quit" command to the `git-fsmonitor--daemon` (if running) - * and wait for it to shutdown. - */ -static int do_as_client__send_stop(void) -{ - struct strbuf answer = STRBUF_INIT; - int ret; - - ret = fsmonitor_ipc__send_command("quit", &answer); - - /* The quit command does not return any response data. */ - strbuf_release(&answer); - - if (ret) - return ret; - - trace2_region_enter("fsm_client", "polling-for-daemon-exit", NULL); - while (fsmonitor_ipc__get_state() == IPC_STATE__LISTENING) - sleep_millisec(50); - trace2_region_leave("fsm_client", "polling-for-daemon-exit", NULL); - - return 0; -} - -static int do_as_client__status(void) -{ - enum ipc_active_state state = fsmonitor_ipc__get_state(); - - switch (state) { - case IPC_STATE__LISTENING: - printf(_("fsmonitor-daemon is watching '%s'\n"), - the_repository->worktree); - return 0; - - default: - printf(_("fsmonitor-daemon is not watching '%s'\n"), - the_repository->worktree); - return 1; - } -} int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) { @@ -80,12 +35,6 @@ int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) argc = parse_options(argc, argv, prefix, options, builtin_fsmonitor__daemon_usage, 0); - if (!strcmp(subcmd, "stop")) - return !!do_as_client__send_stop(); - - if (!strcmp(subcmd, "status")) - return !!do_as_client__status(); - die(_("Unhandled subcommand '%s'"), subcmd); } From 669c95526b6edf8e73c57ced416ff0502d830bb8 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:37 -0400 Subject: [PATCH 039/110] fixup! fsmonitor--daemon: add a built-in fsmonitor daemon This reverts commit b441b3720edaeae8b23e3e29df06557278257c2f. Signed-off-by: Jeff Hostetler --- .gitignore | 1 - Makefile | 1 - builtin.h | 1 - builtin/fsmonitor--daemon.c | 53 ------------------------------------- git.c | 1 - 5 files changed, 57 deletions(-) delete mode 100644 builtin/fsmonitor--daemon.c diff --git a/.gitignore b/.gitignore index 4baba472aa8261..311841f9bed577 100644 --- a/.gitignore +++ b/.gitignore @@ -72,7 +72,6 @@ /git-format-patch /git-fsck /git-fsck-objects -/git-fsmonitor--daemon /git-gc /git-get-tar-commit-id /git-grep diff --git a/Makefile b/Makefile index f62752c1cccdce..e595e9539d3e41 100644 --- a/Makefile +++ b/Makefile @@ -1109,7 +1109,6 @@ BUILTIN_OBJS += builtin/fmt-merge-msg.o BUILTIN_OBJS += builtin/for-each-ref.o BUILTIN_OBJS += builtin/for-each-repo.o BUILTIN_OBJS += builtin/fsck.o -BUILTIN_OBJS += builtin/fsmonitor--daemon.o BUILTIN_OBJS += builtin/gc.o BUILTIN_OBJS += builtin/get-tar-commit-id.o BUILTIN_OBJS += builtin/grep.o diff --git a/builtin.h b/builtin.h index 2470d1cd3a267a..16ecd5586f0bee 100644 --- a/builtin.h +++ b/builtin.h @@ -159,7 +159,6 @@ int cmd_for_each_ref(int argc, const char **argv, const char *prefix); int cmd_for_each_repo(int argc, const char **argv, const char *prefix); int cmd_format_patch(int argc, const char **argv, const char *prefix); int cmd_fsck(int argc, const char **argv, const char *prefix); -int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix); int cmd_gc(int argc, const char **argv, const char *prefix); int cmd_get_tar_commit_id(int argc, const char **argv, const char *prefix); int cmd_grep(int argc, const char **argv, const char *prefix); diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c deleted file mode 100644 index df2bad531118c8..00000000000000 --- a/builtin/fsmonitor--daemon.c +++ /dev/null @@ -1,53 +0,0 @@ -#include "builtin.h" -#include "config.h" -#include "parse-options.h" -#include "fsmonitor.h" -#include "fsmonitor-ipc.h" -#include "simple-ipc.h" -#include "khash.h" - -static const char * const builtin_fsmonitor__daemon_usage[] = { - NULL -}; - -#ifdef HAVE_FSMONITOR_DAEMON_BACKEND - -int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) -{ - const char *subcmd; - - struct option options[] = { - OPT_END() - }; - - if (argc < 2) - usage_with_options(builtin_fsmonitor__daemon_usage, options); - - if (argc == 2 && !strcmp(argv[1], "-h")) - usage_with_options(builtin_fsmonitor__daemon_usage, options); - - git_config(git_default_config, NULL); - - subcmd = argv[1]; - argv--; - argc++; - - argc = parse_options(argc, argv, prefix, options, - builtin_fsmonitor__daemon_usage, 0); - - die(_("Unhandled subcommand '%s'"), subcmd); -} - -#else -int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) -{ - struct option options[] = { - OPT_END() - }; - - if (argc == 2 && !strcmp(argv[1], "-h")) - usage_with_options(builtin_fsmonitor__daemon_usage, options); - - die(_("fsmonitor--daemon not supported on this platform")); -} -#endif diff --git a/git.c b/git.c index c6160f4a88612c..18bed9a99647aa 100644 --- a/git.c +++ b/git.c @@ -533,7 +533,6 @@ static struct cmd_struct commands[] = { { "format-patch", cmd_format_patch, RUN_SETUP }, { "fsck", cmd_fsck, RUN_SETUP }, { "fsck-objects", cmd_fsck, RUN_SETUP }, - { "fsmonitor--daemon", cmd_fsmonitor__daemon, RUN_SETUP }, { "gc", cmd_gc, RUN_SETUP }, { "get-tar-commit-id", cmd_get_tar_commit_id, NO_PARSEOPT }, { "grep", cmd_grep, RUN_SETUP_GENTLY }, From c6e29c27dc749a5db6e20befd2619b7b79858fef Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:40 -0400 Subject: [PATCH 040/110] fixup! fsmonitor: use IPC to query the builtin FSMonitor daemon This reverts commit abc1c46df9fb45358d8541c7e6a9bc53e176d5bc. Signed-off-by: Jeff Hostetler --- fsmonitor.c | 34 ++-------------------------------- 1 file changed, 2 insertions(+), 32 deletions(-) diff --git a/fsmonitor.c b/fsmonitor.c index af3c134e7d7082..210d2df9d17fdc 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -254,37 +254,8 @@ void refresh_fsmonitor(struct index_state *istate) trace_printf_key(&trace_fsmonitor, "refresh fsmonitor"); if (fsm_mode == FSMONITOR_MODE_IPC) { - query_success = !fsmonitor_ipc__send_query( - istate->fsmonitor_last_update ? - istate->fsmonitor_last_update : "builtin:fake", - &query_result); - if (query_success) { - /* - * The response contains a series of nul terminated - * strings. The first is the new token. - * - * Use `char *buf` as an interlude to trick the CI - * static analysis to let us use `strbuf_addstr()` - * here (and only copy the token) rather than - * `strbuf_addbuf()`. - */ - buf = query_result.buf; - strbuf_addstr(&last_update_token, buf); - bol = last_update_token.len + 1; - } else { - /* - * The builtin daemon is not available on this - * platform -OR- we failed to get a response. - * - * Generate a fake token (rather than a V1 - * timestamp) for the index extension. (If - * they switch back to the hook API, we don't - * want ambiguous state.) - */ - strbuf_addstr(&last_update_token, "builtin:fake"); - } - - goto apply_results; + /* TODO */ + return; } assert(fsm_mode == FSMONITOR_MODE_HOOK); @@ -350,7 +321,6 @@ void refresh_fsmonitor(struct index_state *istate) query_success ? "success" : "failure"); } -apply_results: /* a fsmonitor process can return '/' to indicate all entries are invalid */ if (query_success && query_result.buf[bol] != '/') { /* Mark all entries returned by the monitor as dirty */ From feea1cbe19a7f3b2434d5c7ed25a103555ffd74c Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:43 -0400 Subject: [PATCH 041/110] fixup! fsmonitor: config settings are repository-specific This reverts commit 81f7337e3dc7748998e41b1434ce230b726d58fe. Signed-off-by: Jeff Hostetler --- Makefile | 1 - builtin/update-index.c | 19 ++------- cache.h | 1 + config.c | 14 ++++++ config.h | 1 + environment.c | 1 + fsmonitor-settings.c | 97 ------------------------------------------ fsmonitor-settings.h | 21 --------- fsmonitor.c | 63 +++++++++++---------------- fsmonitor.h | 18 ++------ repo-settings.c | 2 - repository.h | 3 -- t/README | 4 +- 13 files changed, 51 insertions(+), 194 deletions(-) delete mode 100644 fsmonitor-settings.c delete mode 100644 fsmonitor-settings.h diff --git a/Makefile b/Makefile index e595e9539d3e41..4f6920d39ebac0 100644 --- a/Makefile +++ b/Makefile @@ -904,7 +904,6 @@ LIB_OBJS += fmt-merge-msg.o LIB_OBJS += fsck.o LIB_OBJS += fsmonitor.o LIB_OBJS += fsmonitor-ipc.o -LIB_OBJS += fsmonitor-settings.o LIB_OBJS += gettext.o LIB_OBJS += gpg-interface.o LIB_OBJS += graph.o diff --git a/builtin/update-index.c b/builtin/update-index.c index 25dae7360a601f..f1f16f2de526d9 100644 --- a/builtin/update-index.c +++ b/builtin/update-index.c @@ -1216,25 +1216,14 @@ int cmd_update_index(int argc, const char **argv, const char *prefix) } if (fsmonitor > 0) { - enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); - - if (fsm_mode == FSMONITOR_MODE_DISABLED) { - warning(_("core.useBuiltinFSMonitor is unset; " - "set it if you really want to enable the " - "builtin fsmonitor")); + if (git_config_get_fsmonitor() == 0) warning(_("core.fsmonitor is unset; " - "set it if you really want to enable the " - "hook-based fsmonitor")); - } + "set it if you really want to " + "enable fsmonitor")); add_fsmonitor(&the_index); report(_("fsmonitor enabled")); } else if (!fsmonitor) { - enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); - if (fsm_mode == FSMONITOR_MODE_IPC) - warning(_("core.useBuiltinFSMonitor is set; " - "remove it if you really want to " - "disable fsmonitor")); - if (fsm_mode == FSMONITOR_MODE_HOOK) + if (git_config_get_fsmonitor() == 1) warning(_("core.fsmonitor is set; " "remove it if you really want to " "disable fsmonitor")); diff --git a/cache.h b/cache.h index eb5903e5fc4229..f13a157d0372fd 100644 --- a/cache.h +++ b/cache.h @@ -990,6 +990,7 @@ extern int core_preload_index; extern int precomposed_unicode; extern int protect_hfs; extern int protect_ntfs; +extern const char *core_fsmonitor; extern int core_apply_sparse_checkout; extern int core_sparse_checkout_cone; diff --git a/config.c b/config.c index 19f1d719c81d57..f33abeab851542 100644 --- a/config.c +++ b/config.c @@ -2517,6 +2517,20 @@ int git_config_get_max_percent_split_change(void) return -1; /* default value */ } +int git_config_get_fsmonitor(void) +{ + if (git_config_get_pathname("core.fsmonitor", &core_fsmonitor)) + core_fsmonitor = getenv("GIT_TEST_FSMONITOR"); + + if (core_fsmonitor && !*core_fsmonitor) + core_fsmonitor = NULL; + + if (core_fsmonitor) + return 1; + + return 0; +} + int git_config_get_index_threads(int *dest) { int is_bool, val; diff --git a/config.h b/config.h index 110c426b082547..a2200f311156c4 100644 --- a/config.h +++ b/config.h @@ -609,6 +609,7 @@ int git_config_get_index_threads(int *dest); int git_config_get_untracked_cache(void); int git_config_get_split_index(void); int git_config_get_max_percent_split_change(void); +int git_config_get_fsmonitor(void); /* This dies if the configured or default date is in the future */ int git_config_get_expiry(const char *key, const char **output); diff --git a/environment.c b/environment.c index e6b66315284b57..d6b22ede7ea288 100644 --- a/environment.c +++ b/environment.c @@ -84,6 +84,7 @@ int protect_hfs = PROTECT_HFS_DEFAULT; #define PROTECT_NTFS_DEFAULT 1 #endif int protect_ntfs = PROTECT_NTFS_DEFAULT; +const char *core_fsmonitor; /* * The character that begins a commented line in user-editable file diff --git a/fsmonitor-settings.c b/fsmonitor-settings.c deleted file mode 100644 index 2770266f5ee1ad..00000000000000 --- a/fsmonitor-settings.c +++ /dev/null @@ -1,97 +0,0 @@ -#include "cache.h" -#include "config.h" -#include "repository.h" -#include "fsmonitor-settings.h" - -/* - * We keep this structure defintion private and have getters - * for all fields so that we can lazy load it as needed. - */ -struct fsmonitor_settings { - enum fsmonitor_mode mode; - char *hook_path; -}; - -void fsm_settings__set_ipc(struct repository *r) -{ - struct fsmonitor_settings *s = r->settings.fsmonitor; - - s->mode = FSMONITOR_MODE_IPC; -} - -void fsm_settings__set_hook(struct repository *r, const char *path) -{ - struct fsmonitor_settings *s = r->settings.fsmonitor; - - s->mode = FSMONITOR_MODE_HOOK; - s->hook_path = strdup(path); -} - -void fsm_settings__set_disabled(struct repository *r) -{ - struct fsmonitor_settings *s = r->settings.fsmonitor; - - s->mode = FSMONITOR_MODE_DISABLED; - FREE_AND_NULL(s->hook_path); -} - -static int check_for_ipc(struct repository *r) -{ - int value; - - if (!repo_config_get_bool(r, "core.usebuiltinfsmonitor", &value) && - value) { - fsm_settings__set_ipc(r); - return 1; - } - - return 0; -} - -static int check_for_hook(struct repository *r) -{ - const char *const_str; - - if (repo_config_get_pathname(r, "core.fsmonitor", &const_str)) - const_str = getenv("GIT_TEST_FSMONITOR"); - - if (const_str && *const_str) { - fsm_settings__set_hook(r, const_str); - return 1; - } - - return 0; -} - -static void lookup_fsmonitor_settings(struct repository *r) -{ - struct fsmonitor_settings *s; - - CALLOC_ARRAY(s, 1); - - r->settings.fsmonitor = s; - - if (check_for_ipc(r)) - return; - - if (check_for_hook(r)) - return; - - fsm_settings__set_disabled(r); -} - -enum fsmonitor_mode fsm_settings__get_mode(struct repository *r) -{ - if (!r->settings.fsmonitor) - lookup_fsmonitor_settings(r); - - return r->settings.fsmonitor->mode; -} - -const char *fsm_settings__get_hook_path(struct repository *r) -{ - if (!r->settings.fsmonitor) - lookup_fsmonitor_settings(r); - - return r->settings.fsmonitor->hook_path; -} diff --git a/fsmonitor-settings.h b/fsmonitor-settings.h deleted file mode 100644 index 50b2923461656c..00000000000000 --- a/fsmonitor-settings.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef FSMONITOR_SETTINGS_H -#define FSMONITOR_SETTINGS_H - -struct repository; - -enum fsmonitor_mode { - FSMONITOR_MODE_DISABLED = 0, - FSMONITOR_MODE_HOOK = 1, /* core.fsmonitor */ - FSMONITOR_MODE_IPC = 2, /* core.useBuiltinFSMonitor */ -}; - -void fsm_settings__set_ipc(struct repository *r); -void fsm_settings__set_hook(struct repository *r, const char *path); -void fsm_settings__set_disabled(struct repository *r); - -enum fsmonitor_mode fsm_settings__get_mode(struct repository *r); -const char *fsm_settings__get_hook_path(struct repository *r); - -struct fsmonitor_settings; - -#endif /* FSMONITOR_SETTINGS_H */ diff --git a/fsmonitor.c b/fsmonitor.c index 210d2df9d17fdc..ab9bfc60b34e31 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -3,7 +3,6 @@ #include "dir.h" #include "ewah/ewok.h" #include "fsmonitor.h" -#include "fsmonitor-ipc.h" #include "run-command.h" #include "strbuf.h" @@ -149,18 +148,15 @@ void write_fsmonitor_extension(struct strbuf *sb, struct index_state *istate) /* * Call the query-fsmonitor hook passing the last update token of the saved results. */ -static int query_fsmonitor_hook(struct repository *r, - int version, - const char *last_update, - struct strbuf *query_result) +static int query_fsmonitor(int version, const char *last_update, struct strbuf *query_result) { struct child_process cp = CHILD_PROCESS_INIT; int result; - if (fsm_settings__get_mode(r) != FSMONITOR_MODE_HOOK) + if (!core_fsmonitor) return -1; - strvec_push(&cp.args, fsm_settings__get_hook_path(r)); + strvec_push(&cp.args, core_fsmonitor); strvec_pushf(&cp.args, "%d", version); strvec_pushf(&cp.args, "%s", last_update); cp.use_shell = 1; @@ -242,28 +238,17 @@ void refresh_fsmonitor(struct index_state *istate) struct strbuf last_update_token = STRBUF_INIT; char *buf; unsigned int i; - struct repository *r = istate->repo ? istate->repo : the_repository; - enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); - if (fsm_mode <= FSMONITOR_MODE_DISABLED || - istate->fsmonitor_has_run_once) + if (!core_fsmonitor || istate->fsmonitor_has_run_once) return; + hook_version = fsmonitor_hook_version(); + istate->fsmonitor_has_run_once = 1; trace_printf_key(&trace_fsmonitor, "refresh fsmonitor"); - - if (fsm_mode == FSMONITOR_MODE_IPC) { - /* TODO */ - return; - } - - assert(fsm_mode == FSMONITOR_MODE_HOOK); - - hook_version = fsmonitor_hook_version(); - /* - * This could be racy so save the date/time now and query_fsmonitor_hook + * This could be racy so save the date/time now and query_fsmonitor * should be inclusive to ensure we don't miss potential changes. */ last_update = getnanotime(); @@ -271,14 +256,13 @@ void refresh_fsmonitor(struct index_state *istate) strbuf_addf(&last_update_token, "%"PRIu64"", last_update); /* - * If we have a last update token, call query_fsmonitor_hook for the set of + * If we have a last update token, call query_fsmonitor for the set of * changes since that token, else assume everything is possibly dirty * and check it all. */ if (istate->fsmonitor_last_update) { if (hook_version == -1 || hook_version == HOOK_INTERFACE_VERSION2) { - query_success = !query_fsmonitor_hook( - r, HOOK_INTERFACE_VERSION2, + query_success = !query_fsmonitor(HOOK_INTERFACE_VERSION2, istate->fsmonitor_last_update, &query_result); if (query_success) { @@ -308,17 +292,13 @@ void refresh_fsmonitor(struct index_state *istate) } if (hook_version == HOOK_INTERFACE_VERSION1) { - query_success = !query_fsmonitor_hook( - r, HOOK_INTERFACE_VERSION1, + query_success = !query_fsmonitor(HOOK_INTERFACE_VERSION1, istate->fsmonitor_last_update, &query_result); } - trace_performance_since(last_update, "fsmonitor process '%s'", - fsm_settings__get_hook_path(r)); - trace_printf_key(&trace_fsmonitor, - "fsmonitor process '%s' returned %s", - fsm_settings__get_hook_path(r), - query_success ? "success" : "failure"); + trace_performance_since(last_update, "fsmonitor process '%s'", core_fsmonitor); + trace_printf_key(&trace_fsmonitor, "fsmonitor process '%s' returned %s", + core_fsmonitor, query_success ? "success" : "failure"); } /* a fsmonitor process can return '/' to indicate all entries are invalid */ @@ -431,8 +411,7 @@ void remove_fsmonitor(struct index_state *istate) void tweak_fsmonitor(struct index_state *istate) { unsigned int i; - struct repository *r = istate->repo ? istate->repo : the_repository; - int fsmonitor_enabled = (fsm_settings__get_mode(r) > FSMONITOR_MODE_DISABLED); + int fsmonitor_enabled = git_config_get_fsmonitor(); if (istate->fsmonitor_dirty) { if (fsmonitor_enabled) { @@ -452,8 +431,16 @@ void tweak_fsmonitor(struct index_state *istate) istate->fsmonitor_dirty = NULL; } - if (fsmonitor_enabled) - add_fsmonitor(istate); - else + switch (fsmonitor_enabled) { + case -1: /* keep: do nothing */ + break; + case 0: /* false */ remove_fsmonitor(istate); + break; + case 1: /* true */ + add_fsmonitor(istate); + break; + default: /* unknown value: do nothing */ + break; + } } diff --git a/fsmonitor.h b/fsmonitor.h index f9201411aa74f4..f20d72631d76b4 100644 --- a/fsmonitor.h +++ b/fsmonitor.h @@ -3,7 +3,6 @@ #include "cache.h" #include "dir.h" -#include "fsmonitor-settings.h" extern struct trace_key trace_fsmonitor; @@ -58,11 +57,7 @@ int fsmonitor_is_trivial_response(const struct strbuf *query_result); */ static inline int is_fsmonitor_refreshed(const struct index_state *istate) { - struct repository *r = istate->repo ? istate->repo : the_repository; - enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); - - return fsm_mode <= FSMONITOR_MODE_DISABLED || - istate->fsmonitor_has_run_once; + return !core_fsmonitor || istate->fsmonitor_has_run_once; } /* @@ -72,11 +67,7 @@ static inline int is_fsmonitor_refreshed(const struct index_state *istate) */ static inline void mark_fsmonitor_valid(struct index_state *istate, struct cache_entry *ce) { - struct repository *r = istate->repo ? istate->repo : the_repository; - enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); - - if (fsm_mode > FSMONITOR_MODE_DISABLED && - !(ce->ce_flags & CE_FSMONITOR_VALID)) { + if (core_fsmonitor && !(ce->ce_flags & CE_FSMONITOR_VALID)) { istate->cache_changed = 1; ce->ce_flags |= CE_FSMONITOR_VALID; trace_printf_key(&trace_fsmonitor, "mark_fsmonitor_clean '%s'", ce->name); @@ -92,10 +83,7 @@ static inline void mark_fsmonitor_valid(struct index_state *istate, struct cache */ static inline void mark_fsmonitor_invalid(struct index_state *istate, struct cache_entry *ce) { - struct repository *r = istate->repo ? istate->repo : the_repository; - enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); - - if (fsm_mode > FSMONITOR_MODE_DISABLED) { + if (core_fsmonitor) { ce->ce_flags &= ~CE_FSMONITOR_VALID; untracked_cache_invalidate_path(istate, ce->name, 1); trace_printf_key(&trace_fsmonitor, "mark_fsmonitor_invalid '%s'", ce->name); diff --git a/repo-settings.c b/repo-settings.c index e69dad1e776772..0cfe8b787db26d 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -26,8 +26,6 @@ void prepare_repo_settings(struct repository *r) UPDATE_DEFAULT_BOOL(r->settings.commit_graph_read_changed_paths, 1); UPDATE_DEFAULT_BOOL(r->settings.gc_write_commit_graph, 1); - r->settings.fsmonitor = NULL; /* lazy loaded */ - if (!repo_config_get_int(r, "index.version", &value)) r->settings.index_version = value; if (!repo_config_get_maybe_bool(r, "core.untrackedcache", &value)) { diff --git a/repository.h b/repository.h index fdc0a818b83c08..3740c93bc0fe27 100644 --- a/repository.h +++ b/repository.h @@ -4,7 +4,6 @@ #include "path.h" struct config_set; -struct fsmonitor_settings; struct git_hash_algo; struct index_state; struct lock_file; @@ -36,8 +35,6 @@ struct repo_settings { int gc_write_commit_graph; int fetch_write_commit_graph; - struct fsmonitor_settings *fsmonitor; /* lazy loaded */ - int index_version; enum untracked_cache_setting core_untracked_cache; diff --git a/t/README b/t/README index eba2be8fee0e2b..02ee9ce143fb5e 100644 --- a/t/README +++ b/t/README @@ -398,8 +398,8 @@ every 'git commit-graph write', as if the `--changed-paths` option was passed in. GIT_TEST_FSMONITOR=$PWD/t7519/fsmonitor-all exercises the fsmonitor -code path for utilizing a (hook based) file system monitor to speed up -detecting new or changed files. +code path for utilizing a file system monitor to speed up detecting +new or changed files. GIT_TEST_INDEX_VERSION= exercises the index read/write code path for the index version specified. Can be set to any valid version From a63209e7c158faeab3c7ad3f192ef1ca33494acc Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:45 -0400 Subject: [PATCH 042/110] fixup! help: include fsmonitor--daemon feature flag in version info This reverts commit bd7a1410d34e9e945e147cc0f86b82e97261cf83. Signed-off-by: Jeff Hostetler --- help.c | 4 ---- t/test-lib.sh | 6 ------ 2 files changed, 10 deletions(-) diff --git a/help.c b/help.c index e22ba1d246a5b0..3c3bdec21356d9 100644 --- a/help.c +++ b/help.c @@ -11,7 +11,6 @@ #include "version.h" #include "refs.h" #include "parse-options.h" -#include "fsmonitor-ipc.h" struct category_description { uint32_t category; @@ -665,9 +664,6 @@ void get_version_info(struct strbuf *buf, int show_build_options) strbuf_addf(buf, "sizeof-size_t: %d\n", (int)sizeof(size_t)); strbuf_addf(buf, "shell-path: %s\n", SHELL_PATH); /* NEEDSWORK: also save and output GIT-BUILD_OPTIONS? */ - - if (fsmonitor_ipc__is_supported()) - strbuf_addstr(buf, "feature: fsmonitor--daemon\n"); } } diff --git a/t/test-lib.sh b/t/test-lib.sh index 964c3634d254ad..fd927bd9b6875e 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -1745,9 +1745,3 @@ test_lazy_prereq REBASE_P ' # Tests that verify the scheduler integration must set this locally # to avoid errors. GIT_TEST_MAINT_SCHEDULER="none:exit 1" - -# Does this platform support `git fsmonitor--daemon` -# -test_lazy_prereq FSMONITOR_DAEMON ' - git version --build-options | grep "feature:" | grep "fsmonitor--daemon" -' From 513dee5fd2446741554bb05aaf19b4831dcf6518 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:47 -0400 Subject: [PATCH 043/110] fixup! fsmonitor-ipc: create client routines for git-fsmonitor--daemon This reverts commit a2f5482b4e8e27a1f8e890a09c6f08721c407ca0. Signed-off-by: Jeff Hostetler --- Makefile | 1 - fsmonitor-ipc.c | 179 ------------------------------------------------ fsmonitor-ipc.h | 48 ------------- 3 files changed, 228 deletions(-) delete mode 100644 fsmonitor-ipc.c delete mode 100644 fsmonitor-ipc.h diff --git a/Makefile b/Makefile index 4f6920d39ebac0..1ee317380c3eab 100644 --- a/Makefile +++ b/Makefile @@ -903,7 +903,6 @@ LIB_OBJS += fetch-pack.o LIB_OBJS += fmt-merge-msg.o LIB_OBJS += fsck.o LIB_OBJS += fsmonitor.o -LIB_OBJS += fsmonitor-ipc.o LIB_OBJS += gettext.o LIB_OBJS += gpg-interface.o LIB_OBJS += graph.o diff --git a/fsmonitor-ipc.c b/fsmonitor-ipc.c deleted file mode 100644 index 417069725209b1..00000000000000 --- a/fsmonitor-ipc.c +++ /dev/null @@ -1,179 +0,0 @@ -#include "cache.h" -#include "fsmonitor.h" -#include "simple-ipc.h" -#include "fsmonitor-ipc.h" -#include "run-command.h" -#include "strbuf.h" -#include "trace2.h" - -#ifdef HAVE_FSMONITOR_DAEMON_BACKEND - -int fsmonitor_ipc__is_supported(void) -{ - return 1; -} - -GIT_PATH_FUNC(fsmonitor_ipc__get_path, "fsmonitor--daemon.ipc") - -enum ipc_active_state fsmonitor_ipc__get_state(void) -{ - return ipc_get_active_state(fsmonitor_ipc__get_path()); -} - -static int spawn_daemon(void) -{ - const char *args[] = { "fsmonitor--daemon", "start", NULL }; - - return run_command_v_opt_tr2(args, RUN_COMMAND_NO_STDIN | RUN_GIT_CMD, - "fsmonitor"); -} - -int fsmonitor_ipc__send_query(const char *since_token, - struct strbuf *answer) -{ - int ret = -1; - int tried_to_spawn = 0; - enum ipc_active_state state = IPC_STATE__OTHER_ERROR; - struct ipc_client_connection *connection = NULL; - struct ipc_client_connect_options options - = IPC_CLIENT_CONNECT_OPTIONS_INIT; - const char *tok = since_token ? since_token : ""; - size_t tok_len = since_token ? strlen(since_token) : 0; - - options.wait_if_busy = 1; - options.wait_if_not_found = 0; - - trace2_region_enter("fsm_client", "query", NULL); - trace2_data_string("fsm_client", NULL, "query/command", tok); - -try_again: - state = ipc_client_try_connect(fsmonitor_ipc__get_path(), &options, - &connection); - - switch (state) { - case IPC_STATE__LISTENING: - ret = ipc_client_send_command_to_connection( - connection, tok, tok_len, answer); - ipc_client_close_connection(connection); - - trace2_data_intmax("fsm_client", NULL, - "query/response-length", answer->len); - - if (fsmonitor_is_trivial_response(answer)) - trace2_data_intmax("fsm_client", NULL, - "query/trivial-response", 1); - - goto done; - - case IPC_STATE__NOT_LISTENING: - ret = error(_("fsmonitor_ipc__send_query: daemon not available")); - goto done; - - case IPC_STATE__PATH_NOT_FOUND: - if (tried_to_spawn) - goto done; - - tried_to_spawn++; - if (spawn_daemon()) - goto done; - - /* - * Try again, but this time give the daemon a chance to - * actually create the pipe/socket. - * - * Granted, the daemon just started so it can't possibly have - * any FS cached yet, so we'll always get a trivial answer. - * BUT the answer should include a new token that can serve - * as the basis for subsequent requests. - */ - options.wait_if_not_found = 1; - goto try_again; - - case IPC_STATE__INVALID_PATH: - ret = error(_("fsmonitor_ipc__send_query: invalid path '%s'"), - fsmonitor_ipc__get_path()); - goto done; - - case IPC_STATE__OTHER_ERROR: - default: - ret = error(_("fsmonitor_ipc__send_query: unspecified error on '%s'"), - fsmonitor_ipc__get_path()); - goto done; - } - -done: - trace2_region_leave("fsm_client", "query", NULL); - - return ret; -} - -int fsmonitor_ipc__send_command(const char *command, - struct strbuf *answer) -{ - struct ipc_client_connection *connection = NULL; - struct ipc_client_connect_options options - = IPC_CLIENT_CONNECT_OPTIONS_INIT; - int ret; - enum ipc_active_state state; - const char *c = command ? command : ""; - size_t c_len = command ? strlen(command) : 0; - - strbuf_reset(answer); - - options.wait_if_busy = 1; - options.wait_if_not_found = 0; - - state = ipc_client_try_connect(fsmonitor_ipc__get_path(), &options, - &connection); - if (state != IPC_STATE__LISTENING) { - die("fsmonitor--daemon is not running"); - return -1; - } - - ret = ipc_client_send_command_to_connection(connection, c, c_len, - answer); - ipc_client_close_connection(connection); - - if (ret == -1) { - die("could not send '%s' command to fsmonitor--daemon", c); - return -1; - } - - return 0; -} - -#else - -/* - * A trivial implementation of the fsmonitor_ipc__ API for unsupported - * platforms. - */ - -int fsmonitor_ipc__is_supported(void) -{ - return 0; -} - -const char *fsmonitor_ipc__get_path(void) -{ - return NULL; -} - -enum ipc_active_state fsmonitor_ipc__get_state(void) -{ - return IPC_STATE__OTHER_ERROR; -} - -int fsmonitor_ipc__send_query(const char *since_token, - struct strbuf *answer) -{ - return -1; -} - -int fsmonitor_ipc__send_command(const char *command, - struct strbuf *answer) -{ - return -1; -} - -#endif diff --git a/fsmonitor-ipc.h b/fsmonitor-ipc.h deleted file mode 100644 index 837c5e5b64ad8c..00000000000000 --- a/fsmonitor-ipc.h +++ /dev/null @@ -1,48 +0,0 @@ -#ifndef FSMONITOR_IPC_H -#define FSMONITOR_IPC_H - -/* - * Returns true if built-in file system monitor daemon is defined - * for this platform. - */ -int fsmonitor_ipc__is_supported(void); - -/* - * Returns the pathname to the IPC named pipe or Unix domain socket - * where a `git-fsmonitor--daemon` process will listen. This is a - * per-worktree value. - * - * Returns NULL if the daemon is not supported on this platform. - */ -const char *fsmonitor_ipc__get_path(void); - -/* - * Try to determine whether there is a `git-fsmonitor--daemon` process - * listening on the IPC pipe/socket. - */ -enum ipc_active_state fsmonitor_ipc__get_state(void); - -/* - * Connect to a `git-fsmonitor--daemon` process via simple-ipc - * and ask for the set of changed files since the given token. - * - * This DOES NOT use the hook interface. - * - * Spawn a daemon process in the background if necessary. - * - * Returns -1 on error; 0 on success. - */ -int fsmonitor_ipc__send_query(const char *since_token, - struct strbuf *answer); - -/* - * Connect to a `git-fsmonitor--daemon` process via simple-ipc and - * send a command verb. If no daemon is available, we DO NOT try to - * start one. - * - * Returns -1 on error; 0 on success. - */ -int fsmonitor_ipc__send_command(const char *command, - struct strbuf *answer); - -#endif /* FSMONITOR_IPC_H */ From 5c2eb208245a3fadad3381d498b87720bbd1d13f Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:50 -0400 Subject: [PATCH 044/110] fixup! fsmonitor--daemon: update fsmonitor documentation This reverts commit 5641d51b02fadf125f5f6c8bb5f7ceabdcc674f1. Signed-off-by: Jeff Hostetler --- Documentation/config/core.txt | 56 ++++++++---------------------- Documentation/git-update-index.txt | 27 +++++++------- Documentation/githooks.txt | 3 +- 3 files changed, 27 insertions(+), 59 deletions(-) diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 323f966a09e6fd..eeb226d1b20784 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -62,50 +62,22 @@ core.protectNTFS:: Defaults to `true` on Windows, and `false` elsewhere. core.fsmonitor:: - If set, this variable contains the pathname of the "fsmonitor" - hook command. -+ -This hook command is used to identify all files that may have changed -since the requested date/time. This information is used to speed up -git by avoiding unnecessary scanning of files that have not changed. -+ -See the "fsmonitor-watchman" section of linkgit:githooks[5]. -+ -Note: The value of this config setting is ignored if the -built-in file system monitor is enabled (see `core.useBuiltinFSMonitor`). + If set, the value of this variable is used as a command which + will identify all files that may have changed since the + requested date/time. This information is used to speed up git by + avoiding unnecessary processing of files that have not changed. + See the "fsmonitor-watchman" section of linkgit:githooks[5]. core.fsmonitorHookVersion:: - Sets the protocol version to be used when invoking the - "fsmonitor" hook. -+ -There are currently versions 1 and 2. When this is not set, -version 2 will be tried first and if it fails then version 1 -will be tried. Version 1 uses a timestamp as input to determine -which files have changes since that time but some monitors -like Watchman have race conditions when used with a timestamp. -Version 2 uses an opaque string so that the monitor can return -something that can be used to determine what files have changed -without race conditions. -+ -Note: The value of this config setting is ignored if the -built-in file system monitor is enabled (see `core.useBuiltinFSMonitor`). - -core.useBuiltinFSMonitor:: - If set to true, enable the built-in file system monitor - daemon for this working directory (linkgit:git-fsmonitor--daemon[1]). -+ -Like hook-based file system monitors, the built-in file system monitor -can speed up Git commands that need to refresh the Git index -(e.g. `git status`) in a working directory with many files. The -built-in monitor eliminates the need to install and maintain an -external third-party tool. -+ -The built-in file system monitor is currently available only on a -limited set of supported platforms. Currently, this includes Windows -and MacOS. -+ -Note: if this config setting is set to `true`, the values of -`core.fsmonitor` and `core.fsmonitorHookVersion` are ignored. + Sets the version of hook that is to be used when calling fsmonitor. + There are currently versions 1 and 2. When this is not set, + version 2 will be tried first and if it fails then version 1 + will be tried. Version 1 uses a timestamp as input to determine + which files have changes since that time but some monitors + like watchman have race conditions when used with a timestamp. + Version 2 uses an opaque string so that the monitor can return + something that can be used to determine what files have changed + without race conditions. core.trustctime:: If false, the ctime differences between the index and the diff --git a/Documentation/git-update-index.txt b/Documentation/git-update-index.txt index c7c31b3fcf9c66..2853f168d97685 100644 --- a/Documentation/git-update-index.txt +++ b/Documentation/git-update-index.txt @@ -498,9 +498,7 @@ FILE SYSTEM MONITOR This feature is intended to speed up git operations for repos that have large working directories. -It enables git to work together with a file system monitor (see -linkgit:git-fsmonitor--daemon[1] -and the +It enables git to work together with a file system monitor (see the "fsmonitor-watchman" section of linkgit:githooks[5]) that can inform it as to what files have been modified. This enables git to avoid having to lstat() every file to find modified files. @@ -510,18 +508,17 @@ performance by avoiding the cost of scanning the entire working directory looking for new files. If you want to enable (or disable) this feature, it is easier to use -the `core.fsmonitor` or `core.useBuiltinFSMonitor` configuration -variable (see linkgit:git-config[1]) than using the `--fsmonitor` -option to `git update-index` in each repository, especially if you -want to do so across all repositories you use, because you can set the -configuration variable in your `$HOME/.gitconfig` just once and have -it affect all repositories you touch. - -When the `core.fsmonitor` or `core.useBuiltinFSMonitor` configuration -variable is changed, the file system monitor is added to or removed -from the index the next time a command reads the index. When -`--[no-]fsmonitor` are used, the file system monitor is immediately -added to or removed from the index. +the `core.fsmonitor` configuration variable (see +linkgit:git-config[1]) than using the `--fsmonitor` option to +`git update-index` in each repository, especially if you want to do so +across all repositories you use, because you can set the configuration +variable in your `$HOME/.gitconfig` just once and have it affect all +repositories you touch. + +When the `core.fsmonitor` configuration variable is changed, the +file system monitor is added to or removed from the index the next time +a command reads the index. When `--[no-]fsmonitor` are used, the file +system monitor is immediately added to or removed from the index. CONFIGURATION ------------- diff --git a/Documentation/githooks.txt b/Documentation/githooks.txt index b7d5e926f7b042..b51959ff9418fd 100644 --- a/Documentation/githooks.txt +++ b/Documentation/githooks.txt @@ -593,8 +593,7 @@ fsmonitor-watchman This hook is invoked when the configuration option `core.fsmonitor` is set to `.git/hooks/fsmonitor-watchman` or `.git/hooks/fsmonitor-watchmanv2` -depending on the version of the hook to use, unless overridden via -`core.useBuiltinFSMonitor` (see linkgit:git-config[1]). +depending on the version of the hook to use. Version 1 takes two arguments, a version (1) and the time in elapsed nanoseconds since midnight, January 1, 1970. From f3c5e26fcf69a6175bc103771279f7a9a21772f0 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:15:52 -0400 Subject: [PATCH 045/110] fixup! fsmonitor--daemon: man page This reverts commit e3e590dbc753a720b975fd76cbf1c39e3ae9279d. Signed-off-by: Jeff Hostetler --- Documentation/git-fsmonitor--daemon.txt | 75 ------------------------- 1 file changed, 75 deletions(-) delete mode 100644 Documentation/git-fsmonitor--daemon.txt diff --git a/Documentation/git-fsmonitor--daemon.txt b/Documentation/git-fsmonitor--daemon.txt deleted file mode 100644 index 154e7684daae07..00000000000000 --- a/Documentation/git-fsmonitor--daemon.txt +++ /dev/null @@ -1,75 +0,0 @@ -git-fsmonitor--daemon(1) -======================== - -NAME ----- -git-fsmonitor--daemon - A Built-in File System Monitor - -SYNOPSIS --------- -[verse] -'git fsmonitor--daemon' start -'git fsmonitor--daemon' run -'git fsmonitor--daemon' stop -'git fsmonitor--daemon' status - -DESCRIPTION ------------ - -A daemon to watch the working directory for file and directory -changes using platform-specific file system notification facilities. - -This daemon communicates directly with commands like `git status` -using the link:technical/api-simple-ipc.html[simple IPC] interface -instead of the slower linkgit:githooks[5] interface. - -This daemon is built into Git so that no third-party tools are -required. - -OPTIONS -------- - -start:: - Starts a daemon in the background. - -run:: - Runs a daemon in the foreground. - -stop:: - Stops the daemon running in the current working - directory, if present. - -status:: - Exits with zero status if a daemon is watching the - current working directory. - -REMARKS -------- - -This daemon is a long running process used to watch a single working -directory and maintain a list of the recently changed files and -directories. Performance of commands such as `git status` can be -increased if they just ask for a summary of changes to the working -directory and can avoid scanning the disk. - -When `core.useBuiltinFSMonitor` is set to `true` (see -linkgit:git-config[1]) commands, such as `git status`, will ask the -daemon for changes and automatically start it (if necessary). - -For more information see the "File System Monitor" section in -linkgit:git-update-index[1]. - -CAVEATS -------- - -The fsmonitor daemon does not currently know about submodules and does -not know to filter out file system events that happen within a -submodule. If fsmonitor daemon is watching a super repo and a file is -modified within the working directory of a submodule, it will report -the change (as happening against the super repo). However, the client -will properly ignore these extra events, so performance may be affected -but it will not cause an incorrect result. - -GIT ---- -Part of the linkgit:git[1] suite From 19ee980052a596c025db50d20c46a96e3298b699 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 Sep 2021 15:58:39 -0400 Subject: [PATCH 046/110] fixup! simple-ipc: preparations for supporting binary messages. This reverts commit 15b44c3f2166408b1b526f0127c8c780ff0644a9. Signed-off-by: Jeff Hostetler --- compat/simple-ipc/ipc-unix-socket.c | 14 +++++------- compat/simple-ipc/ipc-win32.c | 14 +++++------- simple-ipc.h | 7 ++---- t/helper/test-simple-ipc.c | 34 ++++++++++------------------- 4 files changed, 23 insertions(+), 46 deletions(-) diff --git a/compat/simple-ipc/ipc-unix-socket.c b/compat/simple-ipc/ipc-unix-socket.c index 4e28857a0a1e22..1927e6ef4bca8e 100644 --- a/compat/simple-ipc/ipc-unix-socket.c +++ b/compat/simple-ipc/ipc-unix-socket.c @@ -168,8 +168,7 @@ void ipc_client_close_connection(struct ipc_client_connection *connection) int ipc_client_send_command_to_connection( struct ipc_client_connection *connection, - const char *message, size_t message_len, - struct strbuf *answer) + const char *message, struct strbuf *answer) { int ret = 0; @@ -177,7 +176,7 @@ int ipc_client_send_command_to_connection( trace2_region_enter("ipc-client", "send-command", NULL); - if (write_packetized_from_buf_no_flush(message, message_len, + if (write_packetized_from_buf_no_flush(message, strlen(message), connection->fd) < 0 || packet_flush_gently(connection->fd) < 0) { ret = error(_("could not send IPC command")); @@ -198,8 +197,7 @@ int ipc_client_send_command_to_connection( int ipc_client_send_command(const char *path, const struct ipc_client_connect_options *options, - const char *message, size_t message_len, - struct strbuf *answer) + const char *message, struct strbuf *answer) { int ret = -1; enum ipc_active_state state; @@ -210,9 +208,7 @@ int ipc_client_send_command(const char *path, if (state != IPC_STATE__LISTENING) return ret; - ret = ipc_client_send_command_to_connection(connection, - message, message_len, - answer); + ret = ipc_client_send_command_to_connection(connection, message, answer); ipc_client_close_connection(connection); @@ -507,7 +503,7 @@ static int worker_thread__do_io( if (ret >= 0) { ret = worker_thread_data->server_data->application_cb( worker_thread_data->server_data->application_data, - buf.buf, buf.len, do_io_reply_callback, &reply_data); + buf.buf, do_io_reply_callback, &reply_data); packet_flush_gently(reply_data.fd); } diff --git a/compat/simple-ipc/ipc-win32.c b/compat/simple-ipc/ipc-win32.c index 8e889d6a506dc0..8dc7bda087da5f 100644 --- a/compat/simple-ipc/ipc-win32.c +++ b/compat/simple-ipc/ipc-win32.c @@ -208,8 +208,7 @@ void ipc_client_close_connection(struct ipc_client_connection *connection) int ipc_client_send_command_to_connection( struct ipc_client_connection *connection, - const char *message, size_t message_len, - struct strbuf *answer) + const char *message, struct strbuf *answer) { int ret = 0; @@ -217,7 +216,7 @@ int ipc_client_send_command_to_connection( trace2_region_enter("ipc-client", "send-command", NULL); - if (write_packetized_from_buf_no_flush(message, message_len, + if (write_packetized_from_buf_no_flush(message, strlen(message), connection->fd) < 0 || packet_flush_gently(connection->fd) < 0) { ret = error(_("could not send IPC command")); @@ -240,8 +239,7 @@ int ipc_client_send_command_to_connection( int ipc_client_send_command(const char *path, const struct ipc_client_connect_options *options, - const char *message, size_t message_len, - struct strbuf *response) + const char *message, struct strbuf *response) { int ret = -1; enum ipc_active_state state; @@ -252,9 +250,7 @@ int ipc_client_send_command(const char *path, if (state != IPC_STATE__LISTENING) return ret; - ret = ipc_client_send_command_to_connection(connection, - message, message_len, - response); + ret = ipc_client_send_command_to_connection(connection, message, response); ipc_client_close_connection(connection); @@ -462,7 +458,7 @@ static int do_io(struct ipc_server_thread_data *server_thread_data) if (ret >= 0) { ret = server_thread_data->server_data->application_cb( server_thread_data->server_data->application_data, - buf.buf, buf.len, do_io_reply_callback, &reply_data); + buf.buf, do_io_reply_callback, &reply_data); packet_flush_gently(reply_data.fd); diff --git a/simple-ipc.h b/simple-ipc.h index 9c7330fcda0056..2c48a5ee004732 100644 --- a/simple-ipc.h +++ b/simple-ipc.h @@ -107,8 +107,7 @@ void ipc_client_close_connection(struct ipc_client_connection *connection); */ int ipc_client_send_command_to_connection( struct ipc_client_connection *connection, - const char *message, size_t message_len, - struct strbuf *answer); + const char *message, struct strbuf *answer); /* * Used by the client to synchronously connect and send and receive a @@ -120,8 +119,7 @@ int ipc_client_send_command_to_connection( */ int ipc_client_send_command(const char *path, const struct ipc_client_connect_options *options, - const char *message, size_t message_len, - struct strbuf *answer); + const char *message, struct strbuf *answer); /* * Simple IPC Server Side API. @@ -146,7 +144,6 @@ typedef int (ipc_server_reply_cb)(struct ipc_server_reply_data *, */ typedef int (ipc_server_application_cb)(void *application_data, const char *request, - size_t request_len, ipc_server_reply_cb *reply_cb, struct ipc_server_reply_data *reply_data); diff --git a/t/helper/test-simple-ipc.c b/t/helper/test-simple-ipc.c index 9134518075097e..42040ef81b1e84 100644 --- a/t/helper/test-simple-ipc.c +++ b/t/helper/test-simple-ipc.c @@ -112,7 +112,7 @@ static int app__slow_command(ipc_server_reply_cb *reply_cb, /* * The client sent a command followed by a (possibly very) large buffer. */ -static int app__sendbytes_command(const char *received, size_t received_len, +static int app__sendbytes_command(const char *received, ipc_server_reply_cb *reply_cb, struct ipc_server_reply_data *reply_data) { @@ -123,13 +123,6 @@ static int app__sendbytes_command(const char *received, size_t received_len, int errs = 0; int ret; - /* - * The test is setup to send: - * "sendbytes" SP - */ - if (received_len < strlen("sendbytes ")) - BUG("received_len is short in app__sendbytes_command"); - if (skip_prefix(received, "sendbytes ", &p)) len_ballast = strlen(p); @@ -167,7 +160,7 @@ static ipc_server_application_cb test_app_cb; * by this application. */ static int test_app_cb(void *application_data, - const char *command, size_t command_len, + const char *command, ipc_server_reply_cb *reply_cb, struct ipc_server_reply_data *reply_data) { @@ -180,7 +173,7 @@ static int test_app_cb(void *application_data, if (application_data != (void*)&my_app_data) BUG("application_cb: application_data pointer wrong"); - if (command_len == 4 && !strncmp(command, "quit", 4)) { + if (!strcmp(command, "quit")) { /* * The client sent a "quit" command. This is an async * request for the server to shutdown. @@ -200,23 +193,22 @@ static int test_app_cb(void *application_data, return SIMPLE_IPC_QUIT; } - if (command_len == 4 && !strncmp(command, "ping", 4)) { + if (!strcmp(command, "ping")) { const char *answer = "pong"; return reply_cb(reply_data, answer, strlen(answer)); } - if (command_len == 3 && !strncmp(command, "big", 3)) + if (!strcmp(command, "big")) return app__big_command(reply_cb, reply_data); - if (command_len == 5 && !strncmp(command, "chunk", 5)) + if (!strcmp(command, "chunk")) return app__chunk_command(reply_cb, reply_data); - if (command_len == 4 && !strncmp(command, "slow", 4)) + if (!strcmp(command, "slow")) return app__slow_command(reply_cb, reply_data); - if (command_len >= 10 && starts_with(command, "sendbytes ")) - return app__sendbytes_command(command, command_len, - reply_cb, reply_data); + if (starts_with(command, "sendbytes ")) + return app__sendbytes_command(command, reply_cb, reply_data); return app__unhandled_command(command, reply_cb, reply_data); } @@ -496,9 +488,7 @@ static int client__send_ipc(void) options.wait_if_busy = 1; options.wait_if_not_found = 0; - if (!ipc_client_send_command(cl_args.path, &options, - command, strlen(command), - &buf)) { + if (!ipc_client_send_command(cl_args.path, &options, command, &buf)) { if (buf.len) { printf("%s\n", buf.buf); fflush(stdout); @@ -566,9 +556,7 @@ static int do_sendbytes(int bytecount, char byte, const char *path, strbuf_addstr(&buf_send, "sendbytes "); strbuf_addchars(&buf_send, byte, bytecount); - if (!ipc_client_send_command(path, options, - buf_send.buf, buf_send.len, - &buf_resp)) { + if (!ipc_client_send_command(path, options, buf_send.buf, &buf_resp)) { strbuf_rtrim(&buf_resp); printf("sent:%c%08d %s\n", byte, bytecount, buf_resp.buf); fflush(stdout); From 54ddae09f9aee6b3476ceee0ffcffb584947d114 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 20 Sep 2021 09:51:59 -0400 Subject: [PATCH 047/110] trace2: add trace2_child_ready() to report on background children Create "child_ready" event to capture the state of a child process created in the background. When a child command is started a "child_start" event is generated in the Trace2 log. For normal synchronous children, a "child_exit" event is later generated when the child exits or is terminated. The two events include information, such as the "child_id" and "pid", to allow post analysis to match-up the command line and exit status. When a child is started in the background (and may outlive the parent process), it is not possible for the parent to emit a "child_exit" event. Create a new "child_ready" event to indicate whether the child was successfully started. Also include the "child_id" and "pid" to allow similar post processing. This will be used in a later commit with the new "start_bg_command()". Signed-off-by: Jeff Hostetler --- Documentation/technical/api-trace2.txt | 40 ++++++++++++++++++++++++++ trace2.c | 31 ++++++++++++++++++++ trace2.h | 25 ++++++++++++++++ trace2/tr2_tgt.h | 5 ++++ trace2/tr2_tgt_event.c | 22 ++++++++++++++ trace2/tr2_tgt_normal.c | 14 +++++++++ trace2/tr2_tgt_perf.c | 15 ++++++++++ 7 files changed, 152 insertions(+) diff --git a/Documentation/technical/api-trace2.txt b/Documentation/technical/api-trace2.txt index 037a91cbcaf8fa..fd6012acba1311 100644 --- a/Documentation/technical/api-trace2.txt +++ b/Documentation/technical/api-trace2.txt @@ -599,6 +599,46 @@ stopping after the waitpid() and includes OS process creation overhead). So this time will be slightly larger than the atexit time reported by the child process itself. +`"child_ready"`:: + This event is generated after the current process has started + a background process and released all handles to it. ++ +------------ +{ + "event":"child_ready", + ... + "child_id":2, + "pid":14708, # child PID + "ready":"ready", # child ready state + "t_rel":0.110605 # observed run-time of child process +} +------------ ++ +Note that the session-id of the child process is not available to +the current/spawning process, so the child's PID is reported here as +a hint for post-processing. (But it is only a hint because the child +process may be a shell script which doesn't have a session-id.) ++ +This event is generated after the child is started in the background +and given a little time to boot up and start working. If the child +startups normally and while the parent is still waiting, the "ready" +field will have the value "ready". +If the child is too slow to start and the parent times out, the field +will have the value "timeout". +If the child starts but the parent is unable to probe it, the field +will have the value "error". ++ +After the parent process emits this event, it will release all of its +handles to the child process and treat the child as a background +daemon. So even if the child does eventually finish booting up, +the parent will not emit an updated event. ++ +Note that the `t_rel` field contains the observed run time in seconds +when the parent released the child process into the background. +The child is assumed to be a long-running daemon process and may +outlive the parent process. So the parent's child event times should +not be compared to the child's atexit times. + `"exec"`:: This event is generated before git attempts to `exec()` another command rather than starting a child process. diff --git a/trace2.c b/trace2.c index 256120c7fd553e..5e3a32049a830b 100644 --- a/trace2.c +++ b/trace2.c @@ -381,6 +381,37 @@ void trace2_child_exit_fl(const char *file, int line, struct child_process *cmd, us_elapsed_child); } +void trace2_child_ready_fl(const char *file, int line, + struct child_process *cmd, + const char *ready) +{ + struct tr2_tgt *tgt_j; + int j; + uint64_t us_now; + uint64_t us_elapsed_absolute; + uint64_t us_elapsed_child; + + if (!trace2_enabled) + return; + + us_now = getnanotime() / 1000; + us_elapsed_absolute = tr2tls_absolute_elapsed(us_now); + + if (cmd->trace2_child_us_start) + us_elapsed_child = us_now - cmd->trace2_child_us_start; + else + us_elapsed_child = 0; + + for_each_wanted_builtin (j, tgt_j) + if (tgt_j->pfn_child_ready_fl) + tgt_j->pfn_child_ready_fl(file, line, + us_elapsed_absolute, + cmd->trace2_child_id, + cmd->pid, + ready, + us_elapsed_child); +} + int trace2_exec_fl(const char *file, int line, const char *exe, const char **argv) { diff --git a/trace2.h b/trace2.h index ede18c2e06373f..c7250a43b9b5d2 100644 --- a/trace2.h +++ b/trace2.h @@ -243,6 +243,31 @@ void trace2_child_exit_fl(const char *file, int line, struct child_process *cmd, #define trace2_child_exit(cmd, code) \ trace2_child_exit_fl(__FILE__, __LINE__, (cmd), (code)) +/** + * Emits a "child_ready" message containing the "child-id" and a flag + * indicating whether the child was considered "ready" when we + * released it. + * + * This function should be called after starting a daemon process in + * the background (and after giving it sufficient time to boot + * up) to indicate that we no longer control or own it. + * + * The "ready" argument should contain one of { "ready", "timeout", + * "error" } to indicate the state of the running daemon when we + * released it. + * + * If the daemon process fails to start or it exits or is terminated + * while we are still waiting for it, the caller should emit a + * regular "child_exit" to report the normal process exit information. + * + */ +void trace2_child_ready_fl(const char *file, int line, + struct child_process *cmd, + const char *ready); + +#define trace2_child_ready(cmd, ready) \ + trace2_child_ready_fl(__FILE__, __LINE__, (cmd), (ready)) + /** * Emit an 'exec' event prior to calling one of exec(), execv(), * execvp(), and etc. On Unix-derived systems, this will be the diff --git a/trace2/tr2_tgt.h b/trace2/tr2_tgt.h index 7b904692123e28..83dba733535133 100644 --- a/trace2/tr2_tgt.h +++ b/trace2/tr2_tgt.h @@ -43,6 +43,10 @@ typedef void(tr2_tgt_evt_child_exit_fl_t)(const char *file, int line, uint64_t us_elapsed_absolute, int cid, int pid, int code, uint64_t us_elapsed_child); +typedef void(tr2_tgt_evt_child_ready_fl_t)(const char *file, int line, + uint64_t us_elapsed_absolute, + int cid, int pid, const char *ready, + uint64_t us_elapsed_child); typedef void(tr2_tgt_evt_thread_start_fl_t)(const char *file, int line, uint64_t us_elapsed_absolute); @@ -113,6 +117,7 @@ struct tr2_tgt { tr2_tgt_evt_alias_fl_t *pfn_alias_fl; tr2_tgt_evt_child_start_fl_t *pfn_child_start_fl; tr2_tgt_evt_child_exit_fl_t *pfn_child_exit_fl; + tr2_tgt_evt_child_ready_fl_t *pfn_child_ready_fl; tr2_tgt_evt_thread_start_fl_t *pfn_thread_start_fl; tr2_tgt_evt_thread_exit_fl_t *pfn_thread_exit_fl; tr2_tgt_evt_exec_fl_t *pfn_exec_fl; diff --git a/trace2/tr2_tgt_event.c b/trace2/tr2_tgt_event.c index 6353e8ad915610..4c93a1b9dbadfa 100644 --- a/trace2/tr2_tgt_event.c +++ b/trace2/tr2_tgt_event.c @@ -363,6 +363,27 @@ static void fn_child_exit_fl(const char *file, int line, jw_release(&jw); } +static void fn_child_ready_fl(const char *file, int line, + uint64_t us_elapsed_absolute, int cid, int pid, + const char *ready, uint64_t us_elapsed_child) +{ + const char *event_name = "child_ready"; + struct json_writer jw = JSON_WRITER_INIT; + double t_rel = (double)us_elapsed_child / 1000000.0; + + jw_object_begin(&jw, 0); + event_fmt_prepare(event_name, file, line, NULL, &jw); + jw_object_intmax(&jw, "child_id", cid); + jw_object_intmax(&jw, "pid", pid); + jw_object_string(&jw, "ready", ready); + jw_object_double(&jw, "t_rel", 6, t_rel); + jw_end(&jw); + + tr2_dst_write_line(&tr2dst_event, &jw.json); + + jw_release(&jw); +} + static void fn_thread_start_fl(const char *file, int line, uint64_t us_elapsed_absolute) { @@ -589,6 +610,7 @@ struct tr2_tgt tr2_tgt_event = { fn_alias_fl, fn_child_start_fl, fn_child_exit_fl, + fn_child_ready_fl, fn_thread_start_fl, fn_thread_exit_fl, fn_exec_fl, diff --git a/trace2/tr2_tgt_normal.c b/trace2/tr2_tgt_normal.c index 31b602c171fc69..d66d2b01001712 100644 --- a/trace2/tr2_tgt_normal.c +++ b/trace2/tr2_tgt_normal.c @@ -233,6 +233,19 @@ static void fn_child_exit_fl(const char *file, int line, strbuf_release(&buf_payload); } +static void fn_child_ready_fl(const char *file, int line, + uint64_t us_elapsed_absolute, int cid, int pid, + const char *ready, uint64_t us_elapsed_child) +{ + struct strbuf buf_payload = STRBUF_INIT; + double elapsed = (double)us_elapsed_child / 1000000.0; + + strbuf_addf(&buf_payload, "child_ready[%d] pid:%d ready:%s elapsed:%.6f", + cid, pid, ready, elapsed); + normal_io_write_fl(file, line, &buf_payload); + strbuf_release(&buf_payload); +} + static void fn_exec_fl(const char *file, int line, uint64_t us_elapsed_absolute, int exec_id, const char *exe, const char **argv) { @@ -311,6 +324,7 @@ struct tr2_tgt tr2_tgt_normal = { fn_alias_fl, fn_child_start_fl, fn_child_exit_fl, + fn_child_ready_fl, NULL, /* thread_start */ NULL, /* thread_exit */ fn_exec_fl, diff --git a/trace2/tr2_tgt_perf.c b/trace2/tr2_tgt_perf.c index a8018f18cc87e8..e0050c45f1bb0c 100644 --- a/trace2/tr2_tgt_perf.c +++ b/trace2/tr2_tgt_perf.c @@ -345,6 +345,20 @@ static void fn_child_exit_fl(const char *file, int line, strbuf_release(&buf_payload); } +static void fn_child_ready_fl(const char *file, int line, + uint64_t us_elapsed_absolute, int cid, int pid, + const char *ready, uint64_t us_elapsed_child) +{ + const char *event_name = "child_ready"; + struct strbuf buf_payload = STRBUF_INIT; + + strbuf_addf(&buf_payload, "[ch%d] pid:%d ready:%s", cid, pid, ready); + + perf_io_write_fl(file, line, event_name, NULL, &us_elapsed_absolute, + &us_elapsed_child, NULL, &buf_payload); + strbuf_release(&buf_payload); +} + static void fn_thread_start_fl(const char *file, int line, uint64_t us_elapsed_absolute) { @@ -537,6 +551,7 @@ struct tr2_tgt tr2_tgt_perf = { fn_alias_fl, fn_child_start_fl, fn_child_exit_fl, + fn_child_ready_fl, fn_thread_start_fl, fn_thread_exit_fl, fn_exec_fl, From f183cc7b0d8430a94579e4f443bfb1968ad6c082 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 20 Apr 2021 14:40:56 -0400 Subject: [PATCH 048/110] simple-ipc: preparations for supporting binary messages. Add `command_len` argument to the Simple IPC API. In my original Simple IPC API, I assumed that the request would always be a null-terminated string of text characters. The `command` argument was just a `const char *`. I found a caller that would like to pass a binary command to the daemon, so I am amending the Simple IPC API to receive `const char *command, size_t command_len` arguments. I considered changing the `command` argument to be a `void *`, but the IPC layer simply passes it to the pkt-line layer which takes a `const char *`, so to avoid confusion I left it as is. Note, the response side has always been a `struct strbuf` which includes the buffer and length, so we already support returning a binary answer. (Yes, it feels a little weird returning a binary buffer in a `strbuf`, but it works.) Signed-off-by: Jeff Hostetler --- compat/simple-ipc/ipc-unix-socket.c | 14 +++++++----- compat/simple-ipc/ipc-win32.c | 14 +++++++----- simple-ipc.h | 7 ++++-- t/helper/test-simple-ipc.c | 34 +++++++++++++++++++---------- 4 files changed, 46 insertions(+), 23 deletions(-) diff --git a/compat/simple-ipc/ipc-unix-socket.c b/compat/simple-ipc/ipc-unix-socket.c index 1927e6ef4bca8e..4e28857a0a1e22 100644 --- a/compat/simple-ipc/ipc-unix-socket.c +++ b/compat/simple-ipc/ipc-unix-socket.c @@ -168,7 +168,8 @@ void ipc_client_close_connection(struct ipc_client_connection *connection) int ipc_client_send_command_to_connection( struct ipc_client_connection *connection, - const char *message, struct strbuf *answer) + const char *message, size_t message_len, + struct strbuf *answer) { int ret = 0; @@ -176,7 +177,7 @@ int ipc_client_send_command_to_connection( trace2_region_enter("ipc-client", "send-command", NULL); - if (write_packetized_from_buf_no_flush(message, strlen(message), + if (write_packetized_from_buf_no_flush(message, message_len, connection->fd) < 0 || packet_flush_gently(connection->fd) < 0) { ret = error(_("could not send IPC command")); @@ -197,7 +198,8 @@ int ipc_client_send_command_to_connection( int ipc_client_send_command(const char *path, const struct ipc_client_connect_options *options, - const char *message, struct strbuf *answer) + const char *message, size_t message_len, + struct strbuf *answer) { int ret = -1; enum ipc_active_state state; @@ -208,7 +210,9 @@ int ipc_client_send_command(const char *path, if (state != IPC_STATE__LISTENING) return ret; - ret = ipc_client_send_command_to_connection(connection, message, answer); + ret = ipc_client_send_command_to_connection(connection, + message, message_len, + answer); ipc_client_close_connection(connection); @@ -503,7 +507,7 @@ static int worker_thread__do_io( if (ret >= 0) { ret = worker_thread_data->server_data->application_cb( worker_thread_data->server_data->application_data, - buf.buf, do_io_reply_callback, &reply_data); + buf.buf, buf.len, do_io_reply_callback, &reply_data); packet_flush_gently(reply_data.fd); } diff --git a/compat/simple-ipc/ipc-win32.c b/compat/simple-ipc/ipc-win32.c index 8dc7bda087da5f..8e889d6a506dc0 100644 --- a/compat/simple-ipc/ipc-win32.c +++ b/compat/simple-ipc/ipc-win32.c @@ -208,7 +208,8 @@ void ipc_client_close_connection(struct ipc_client_connection *connection) int ipc_client_send_command_to_connection( struct ipc_client_connection *connection, - const char *message, struct strbuf *answer) + const char *message, size_t message_len, + struct strbuf *answer) { int ret = 0; @@ -216,7 +217,7 @@ int ipc_client_send_command_to_connection( trace2_region_enter("ipc-client", "send-command", NULL); - if (write_packetized_from_buf_no_flush(message, strlen(message), + if (write_packetized_from_buf_no_flush(message, message_len, connection->fd) < 0 || packet_flush_gently(connection->fd) < 0) { ret = error(_("could not send IPC command")); @@ -239,7 +240,8 @@ int ipc_client_send_command_to_connection( int ipc_client_send_command(const char *path, const struct ipc_client_connect_options *options, - const char *message, struct strbuf *response) + const char *message, size_t message_len, + struct strbuf *response) { int ret = -1; enum ipc_active_state state; @@ -250,7 +252,9 @@ int ipc_client_send_command(const char *path, if (state != IPC_STATE__LISTENING) return ret; - ret = ipc_client_send_command_to_connection(connection, message, response); + ret = ipc_client_send_command_to_connection(connection, + message, message_len, + response); ipc_client_close_connection(connection); @@ -458,7 +462,7 @@ static int do_io(struct ipc_server_thread_data *server_thread_data) if (ret >= 0) { ret = server_thread_data->server_data->application_cb( server_thread_data->server_data->application_data, - buf.buf, do_io_reply_callback, &reply_data); + buf.buf, buf.len, do_io_reply_callback, &reply_data); packet_flush_gently(reply_data.fd); diff --git a/simple-ipc.h b/simple-ipc.h index 2c48a5ee004732..9c7330fcda0056 100644 --- a/simple-ipc.h +++ b/simple-ipc.h @@ -107,7 +107,8 @@ void ipc_client_close_connection(struct ipc_client_connection *connection); */ int ipc_client_send_command_to_connection( struct ipc_client_connection *connection, - const char *message, struct strbuf *answer); + const char *message, size_t message_len, + struct strbuf *answer); /* * Used by the client to synchronously connect and send and receive a @@ -119,7 +120,8 @@ int ipc_client_send_command_to_connection( */ int ipc_client_send_command(const char *path, const struct ipc_client_connect_options *options, - const char *message, struct strbuf *answer); + const char *message, size_t message_len, + struct strbuf *answer); /* * Simple IPC Server Side API. @@ -144,6 +146,7 @@ typedef int (ipc_server_reply_cb)(struct ipc_server_reply_data *, */ typedef int (ipc_server_application_cb)(void *application_data, const char *request, + size_t request_len, ipc_server_reply_cb *reply_cb, struct ipc_server_reply_data *reply_data); diff --git a/t/helper/test-simple-ipc.c b/t/helper/test-simple-ipc.c index 42040ef81b1e84..9134518075097e 100644 --- a/t/helper/test-simple-ipc.c +++ b/t/helper/test-simple-ipc.c @@ -112,7 +112,7 @@ static int app__slow_command(ipc_server_reply_cb *reply_cb, /* * The client sent a command followed by a (possibly very) large buffer. */ -static int app__sendbytes_command(const char *received, +static int app__sendbytes_command(const char *received, size_t received_len, ipc_server_reply_cb *reply_cb, struct ipc_server_reply_data *reply_data) { @@ -123,6 +123,13 @@ static int app__sendbytes_command(const char *received, int errs = 0; int ret; + /* + * The test is setup to send: + * "sendbytes" SP + */ + if (received_len < strlen("sendbytes ")) + BUG("received_len is short in app__sendbytes_command"); + if (skip_prefix(received, "sendbytes ", &p)) len_ballast = strlen(p); @@ -160,7 +167,7 @@ static ipc_server_application_cb test_app_cb; * by this application. */ static int test_app_cb(void *application_data, - const char *command, + const char *command, size_t command_len, ipc_server_reply_cb *reply_cb, struct ipc_server_reply_data *reply_data) { @@ -173,7 +180,7 @@ static int test_app_cb(void *application_data, if (application_data != (void*)&my_app_data) BUG("application_cb: application_data pointer wrong"); - if (!strcmp(command, "quit")) { + if (command_len == 4 && !strncmp(command, "quit", 4)) { /* * The client sent a "quit" command. This is an async * request for the server to shutdown. @@ -193,22 +200,23 @@ static int test_app_cb(void *application_data, return SIMPLE_IPC_QUIT; } - if (!strcmp(command, "ping")) { + if (command_len == 4 && !strncmp(command, "ping", 4)) { const char *answer = "pong"; return reply_cb(reply_data, answer, strlen(answer)); } - if (!strcmp(command, "big")) + if (command_len == 3 && !strncmp(command, "big", 3)) return app__big_command(reply_cb, reply_data); - if (!strcmp(command, "chunk")) + if (command_len == 5 && !strncmp(command, "chunk", 5)) return app__chunk_command(reply_cb, reply_data); - if (!strcmp(command, "slow")) + if (command_len == 4 && !strncmp(command, "slow", 4)) return app__slow_command(reply_cb, reply_data); - if (starts_with(command, "sendbytes ")) - return app__sendbytes_command(command, reply_cb, reply_data); + if (command_len >= 10 && starts_with(command, "sendbytes ")) + return app__sendbytes_command(command, command_len, + reply_cb, reply_data); return app__unhandled_command(command, reply_cb, reply_data); } @@ -488,7 +496,9 @@ static int client__send_ipc(void) options.wait_if_busy = 1; options.wait_if_not_found = 0; - if (!ipc_client_send_command(cl_args.path, &options, command, &buf)) { + if (!ipc_client_send_command(cl_args.path, &options, + command, strlen(command), + &buf)) { if (buf.len) { printf("%s\n", buf.buf); fflush(stdout); @@ -556,7 +566,9 @@ static int do_sendbytes(int bytecount, char byte, const char *path, strbuf_addstr(&buf_send, "sendbytes "); strbuf_addchars(&buf_send, byte, bytecount); - if (!ipc_client_send_command(path, options, buf_send.buf, &buf_resp)) { + if (!ipc_client_send_command(path, options, + buf_send.buf, buf_send.len, + &buf_resp)) { strbuf_rtrim(&buf_resp); printf("sent:%c%08d %s\n", byte, bytecount, buf_resp.buf); fflush(stdout); From a0a6ebeb6870844038cbbea2a829c0b3df652402 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 19 Aug 2021 11:10:46 -0400 Subject: [PATCH 049/110] simple-ipc: move definition of ipc_active_state outside of ifdef MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From: Carlo Marcelo Arenas Belón Move the declartion of the `enum ipc_active_state` type outside of the SUPPORTS_SIMPLE_IPC ifdef. A later commit will introduce the `fsmonitor_ipc__*()` API and stub in a "mock" implementation that requires this enum in some function signatures. Signed-off-by: Carlo Marcelo Arenas Belón Signed-off-by: Jeff Hostetler --- simple-ipc.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/simple-ipc.h b/simple-ipc.h index 9c7330fcda0056..b396293bdfc553 100644 --- a/simple-ipc.h +++ b/simple-ipc.h @@ -5,13 +5,6 @@ * See Documentation/technical/api-simple-ipc.txt */ -#ifdef SUPPORTS_SIMPLE_IPC -#include "pkt-line.h" - -/* - * Simple IPC Client Side API. - */ - enum ipc_active_state { /* * The pipe/socket exists and the daemon is waiting for connections. @@ -43,6 +36,13 @@ enum ipc_active_state { IPC_STATE__OTHER_ERROR, }; +#ifdef SUPPORTS_SIMPLE_IPC +#include "pkt-line.h" + +/* + * Simple IPC Client Side API. + */ + struct ipc_client_connect_options { /* * Spin under timeout if the server is running but can't From e5a1d91095ac4635b17cb4b26095e88eb33ae2c1 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 28 Jul 2021 14:04:58 -0400 Subject: [PATCH 050/110] simple-ipc/ipc-win32: add trace2 debugging Create "ipc-debug" category events to log unexpected errors when creating Simple-IPC connections. Signed-off-by: Jeff Hostetler --- compat/simple-ipc/ipc-win32.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/compat/simple-ipc/ipc-win32.c b/compat/simple-ipc/ipc-win32.c index 8e889d6a506dc0..a8dd46bd9223f5 100644 --- a/compat/simple-ipc/ipc-win32.c +++ b/compat/simple-ipc/ipc-win32.c @@ -49,6 +49,9 @@ static enum ipc_active_state get_active_state(wchar_t *pipe_path) if (GetLastError() == ERROR_FILE_NOT_FOUND) return IPC_STATE__PATH_NOT_FOUND; + trace2_data_intmax("ipc-debug", NULL, "getstate/waitpipe/gle", + (intmax_t)GetLastError()); + return IPC_STATE__OTHER_ERROR; } @@ -109,9 +112,15 @@ static enum ipc_active_state connect_to_server( t_start_ms = (DWORD)(getnanotime() / 1000000); if (!WaitNamedPipeW(wpath, timeout_ms)) { - if (GetLastError() == ERROR_SEM_TIMEOUT) + DWORD gleWait = GetLastError(); + + if (gleWait == ERROR_SEM_TIMEOUT) return IPC_STATE__NOT_LISTENING; + trace2_data_intmax("ipc-debug", NULL, + "connect/waitpipe/gle", + (intmax_t)gleWait); + return IPC_STATE__OTHER_ERROR; } @@ -133,17 +142,31 @@ static enum ipc_active_state connect_to_server( break; /* try again */ default: + trace2_data_intmax("ipc-debug", NULL, + "connect/createfile/gle", + (intmax_t)gle); + return IPC_STATE__OTHER_ERROR; } } if (!SetNamedPipeHandleState(hPipe, &mode, NULL, NULL)) { + gle = GetLastError(); + trace2_data_intmax("ipc-debug", NULL, + "connect/setpipestate/gle", + (intmax_t)gle); + CloseHandle(hPipe); return IPC_STATE__OTHER_ERROR; } *pfd = _open_osfhandle((intptr_t)hPipe, O_RDWR|O_BINARY); if (*pfd < 0) { + gle = GetLastError(); + trace2_data_intmax("ipc-debug", NULL, + "connect/openosfhandle/gle", + (intmax_t)gle); + CloseHandle(hPipe); return IPC_STATE__OTHER_ERROR; } From e3ac087f25d91647ec7d821ab8d7302b117821ad Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 26 Aug 2021 13:56:44 -0400 Subject: [PATCH 051/110] simple-ipc/ipc-win32: add Windows ACL to named pipe Set an ACL on the named pipe to allow the well-known group EVERYONE to read and write to the IPC server's named pipe. In the event that the daemon was started with elevation, allow non-elevated clients to communicate with the daemon. Signed-off-by: Jeff Hostetler --- compat/simple-ipc/ipc-win32.c | 140 +++++++++++++++++++++++++++++++--- 1 file changed, 129 insertions(+), 11 deletions(-) diff --git a/compat/simple-ipc/ipc-win32.c b/compat/simple-ipc/ipc-win32.c index a8dd46bd9223f5..20ea7b65e0ba63 100644 --- a/compat/simple-ipc/ipc-win32.c +++ b/compat/simple-ipc/ipc-win32.c @@ -3,6 +3,8 @@ #include "strbuf.h" #include "pkt-line.h" #include "thread-utils.h" +#include "accctrl.h" +#include "aclapi.h" #ifndef SUPPORTS_SIMPLE_IPC /* @@ -592,11 +594,132 @@ static void *server_thread_proc(void *_server_thread_data) return NULL; } +/* + * We need to build a Windows "SECURITY_ATTRIBUTES" object and use it + * to apply an ACL when we create the initial instance of the Named + * Pipe. The construction is somewhat involved and consists of + * several sequential steps and intermediate objects. + * + * We use this structure to hold these intermediate pointers so that + * we can free them as a group. (It is unclear from the docs whether + * some of these intermediate pointers can be freed before we are + * finished using the "lpSA" member.) + */ +struct my_sa_data +{ + PSID pEveryoneSID; + PACL pACL; + PSECURITY_DESCRIPTOR pSD; + LPSECURITY_ATTRIBUTES lpSA; +}; + +static void init_sa(struct my_sa_data *d) +{ + memset(d, 0, sizeof(*d)); +} + +static void release_sa(struct my_sa_data *d) +{ + if (d->pEveryoneSID) + FreeSid(d->pEveryoneSID); + if (d->pACL) + LocalFree(d->pACL); + if (d->pSD) + LocalFree(d->pSD); + if (d->lpSA) + LocalFree(d->lpSA); + + memset(d, 0, sizeof(*d)); +} + +/* + * Create SECURITY_ATTRIBUTES to apply to the initial named pipe. The + * creator of the first server instance gets to set the ACLs on it. + * + * We allow the well-known group `EVERYONE` to have read+write access + * to the named pipe so that clients can send queries to the daemon + * and receive the response. + * + * Normally, this is not necessary since the daemon is usually + * automatically started by a foreground command like `git status`, + * but in those cases where an elevated Git command started the daemon + * (such that the daemon itself runs with elevation), we need to add + * the ACL so that non-elevated commands can write to it. + * + * The following document was helpful: + * https://docs.microsoft.com/en-us/windows/win32/secauthz/creating-a-security-descriptor-for-a-new-object-in-c-- + * + * Returns d->lpSA set to a SA or NULL. + */ +static LPSECURITY_ATTRIBUTES get_sa(struct my_sa_data *d) +{ + SID_IDENTIFIER_AUTHORITY sid_auth_world = SECURITY_WORLD_SID_AUTHORITY; +#define NR_EA (1) + EXPLICIT_ACCESS ea[NR_EA]; + DWORD dwResult; + + if (!AllocateAndInitializeSid(&sid_auth_world, 1, + SECURITY_WORLD_RID, 0,0,0,0,0,0,0, + &d->pEveryoneSID)) { + DWORD gle = GetLastError(); + trace2_data_intmax("ipc-debug", NULL, "alloc-world-sid/gle", + (intmax_t)gle); + goto fail; + } + + memset(ea, 0, NR_EA * sizeof(EXPLICIT_ACCESS)); + + ea[0].grfAccessPermissions = GENERIC_READ | GENERIC_WRITE; + ea[0].grfAccessMode = SET_ACCESS; + ea[0].grfInheritance = NO_INHERITANCE; + ea[0].Trustee.MultipleTrusteeOperation = NO_MULTIPLE_TRUSTEE; + ea[0].Trustee.TrusteeForm = TRUSTEE_IS_SID; + ea[0].Trustee.TrusteeType = TRUSTEE_IS_WELL_KNOWN_GROUP; + ea[0].Trustee.ptstrName = (LPTSTR)d->pEveryoneSID; + + dwResult = SetEntriesInAcl(NR_EA, ea, NULL, &d->pACL); + if (dwResult != ERROR_SUCCESS) { + DWORD gle = GetLastError(); + trace2_data_intmax("ipc-debug", NULL, "set-acl-entry/gle", + (intmax_t)gle); + trace2_data_intmax("ipc-debug", NULL, "set-acl-entry/dw", + (intmax_t)dwResult); + goto fail; + } + + d->pSD = (PSECURITY_DESCRIPTOR)LocalAlloc( + LPTR, SECURITY_DESCRIPTOR_MIN_LENGTH); + if (!InitializeSecurityDescriptor(d->pSD, SECURITY_DESCRIPTOR_REVISION)) { + DWORD gle = GetLastError(); + trace2_data_intmax("ipc-debug", NULL, "init-sd/gle", (intmax_t)gle); + goto fail; + } + + if (!SetSecurityDescriptorDacl(d->pSD, TRUE, d->pACL, FALSE)) { + DWORD gle = GetLastError(); + trace2_data_intmax("ipc-debug", NULL, "set-sd-dacl/gle", (intmax_t)gle); + goto fail; + } + + d->lpSA = (LPSECURITY_ATTRIBUTES)LocalAlloc(LPTR, sizeof(SECURITY_ATTRIBUTES)); + d->lpSA->nLength = sizeof(SECURITY_ATTRIBUTES); + d->lpSA->lpSecurityDescriptor = d->pSD; + d->lpSA->bInheritHandle = FALSE; + + return d->lpSA; + +fail: + release_sa(d); + return NULL; +} + static HANDLE create_new_pipe(wchar_t *wpath, int is_first) { HANDLE hPipe; DWORD dwOpenMode, dwPipeMode; - LPSECURITY_ATTRIBUTES lpsa = NULL; + struct my_sa_data my_sa_data; + + init_sa(&my_sa_data); dwOpenMode = PIPE_ACCESS_INBOUND | PIPE_ACCESS_OUTBOUND | FILE_FLAG_OVERLAPPED; @@ -612,20 +735,15 @@ static HANDLE create_new_pipe(wchar_t *wpath, int is_first) * set the ACL / Security Attributes on the named * pipe; subsequent instances inherit and cannot * change them. - * - * TODO Should we allow the application layer to - * specify security attributes, such as `LocalService` - * or `LocalSystem`, when we create the named pipe? - * This question is probably not important when the - * daemon is started by a foreground user process and - * only needs to talk to the current user, but may be - * if the daemon is run via the Control Panel as a - * System Service. */ + get_sa(&my_sa_data); } hPipe = CreateNamedPipeW(wpath, dwOpenMode, dwPipeMode, - PIPE_UNLIMITED_INSTANCES, 1024, 1024, 0, lpsa); + PIPE_UNLIMITED_INSTANCES, 1024, 1024, 0, + my_sa_data.lpSA); + + release_sa(&my_sa_data); return hPipe; } From e96c86e4c2f7080800a1d643fb081110e1400e96 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 12 Aug 2021 08:21:45 -0400 Subject: [PATCH 052/110] run-command: create start_bg_command Create a variation of `run_command()` and `start_command()` to launch a command into the background and optionally wait for it to become "ready" before returning. Signed-off-by: Jeff Hostetler --- run-command.c | 129 ++++++++++++++++++++++++++++++++++++++++++++++++++ run-command.h | 57 ++++++++++++++++++++++ 2 files changed, 186 insertions(+) diff --git a/run-command.c b/run-command.c index f72e72cce73f1a..4512ad69e444f9 100644 --- a/run-command.c +++ b/run-command.c @@ -1903,3 +1903,132 @@ void prepare_other_repo_env(struct strvec *env_array, const char *new_git_dir) } strvec_pushf(env_array, "%s=%s", GIT_DIR_ENVIRONMENT, new_git_dir); } + +enum start_bg_result start_bg_command(struct child_process *cmd, + start_bg_wait_cb *wait_cb, + void *cb_data, + unsigned int timeout_sec) +{ + enum start_bg_result sbgr = SBGR_ERROR; + int ret; + int wait_status; + pid_t pid_seen; + time_t time_limit; + + /* + * We do not allow clean-on-exit because the child process + * should persist in the background and possibly/probably + * after this process exits. So we don't want to kill the + * child during our atexit routine. + */ + if (cmd->clean_on_exit) + BUG("start_bg_command() does not allow non-zero clean_on_exit"); + + if (!cmd->trace2_child_class) + cmd->trace2_child_class = "background"; + + ret = start_command(cmd); + if (ret) { + /* + * We assume that if `start_command()` fails, we + * either get a complete `trace2_child_start() / + * trace2_child_exit()` pair or it fails before the + * `trace2_child_start()` is emitted, so we do not + * need to worry about it here. + * + * We also assume that `start_command()` does not add + * us to the cleanup list. And that it calls + * calls `child_process_clear()`. + */ + sbgr = SBGR_ERROR; + goto done; + } + + time(&time_limit); + time_limit += timeout_sec; + +wait: + pid_seen = waitpid(cmd->pid, &wait_status, WNOHANG); + + if (!pid_seen) { + /* + * The child is currently running. Ask the callback + * if the child is ready to do work or whether we + * should keep waiting for it to boot up. + */ + ret = (*wait_cb)(cmd, cb_data); + if (!ret) { + /* + * The child is running and "ready". + */ + trace2_child_ready(cmd, "ready"); + sbgr = SBGR_READY; + goto done; + } else if (ret > 0) { + /* + * The callback said to give it more time to boot up + * (subject to our timeout limit). + */ + time_t now; + + time(&now); + if (now < time_limit) + goto wait; + + /* + * Our timeout has expired. We don't try to + * kill the child, but rather let it continue + * (hopefully) trying to startup. + */ + trace2_child_ready(cmd, "timeout"); + sbgr = SBGR_TIMEOUT; + goto done; + } else { + /* + * The cb gave up on this child. It is still running, + * but our cb got an error trying to probe it. + */ + trace2_child_ready(cmd, "error"); + sbgr = SBGR_CB_ERROR; + goto done; + } + } + + else if (pid_seen == cmd->pid) { + int child_code = -1; + + /* + * The child started, but exited or was terminated + * before becoming "ready". + * + * We try to match the behavior of `wait_or_whine()` + * WRT the handling of WIFSIGNALED() and WIFEXITED() + * and convert the child's status to a return code for + * tracing purposes and emit the `trace2_child_exit()` + * event. + * + * We do not want the wait_or_whine() error message + * because we will be called by client-side library + * routines. + */ + if (WIFEXITED(wait_status)) + child_code = WEXITSTATUS(wait_status); + else if (WIFSIGNALED(wait_status)) + child_code = WTERMSIG(wait_status) + 128; + trace2_child_exit(cmd, child_code); + + sbgr = SBGR_DIED; + goto done; + } + + else if (pid_seen < 0 && errno == EINTR) + goto wait; + + trace2_child_exit(cmd, -1); + sbgr = SBGR_ERROR; + +done: + child_process_clear(cmd); + invalidate_lstat_cache(); + return sbgr; +} diff --git a/run-command.h b/run-command.h index af1296769f9862..17b1b80c3d7667 100644 --- a/run-command.h +++ b/run-command.h @@ -496,4 +496,61 @@ int run_processes_parallel_tr2(int n, get_next_task_fn, start_failure_fn, */ void prepare_other_repo_env(struct strvec *env_array, const char *new_git_dir); +/** + * Possible return values for start_bg_command(). + */ +enum start_bg_result { + /* child process is "ready" */ + SBGR_READY = 0, + + /* child process could not be started */ + SBGR_ERROR, + + /* callback error when testing for "ready" */ + SBGR_CB_ERROR, + + /* timeout expired waiting for child to become "ready" */ + SBGR_TIMEOUT, + + /* child process exited or was signalled before becomming "ready" */ + SBGR_DIED, +}; + +/** + * Callback used by start_bg_command() to ask whether the + * child process is ready or needs more time to become "ready". + * + * The callback will receive the cmd and cb_data arguments given to + * start_bg_command(). + * + * Returns 1 is child needs more time (subject to the requested timeout). + * Returns 0 if child is "ready". + * Returns -1 on any error and cause start_bg_command() to also error out. + */ +typedef int(start_bg_wait_cb)(const struct child_process *cmd, void *cb_data); + +/** + * Start a command in the background. Wait long enough for the child + * to become "ready" (as defined by the provided callback). Capture + * immediate errors (like failure to start) and any immediate exit + * status (such as a shutdown/signal before the child became "ready") + * and return this like start_command(). + * + * We run a custom wait loop using the provided callback to wait for + * the child to start and become "ready". This is limited by the given + * timeout value. + * + * If the child does successfully start and become "ready", we orphan + * it into the background. + * + * The caller must not call finish_command(). + * + * The opaque cb_data argument will be forwarded to the callback for + * any instance data that it might require. This may be NULL. + */ +enum start_bg_result start_bg_command(struct child_process *cmd, + start_bg_wait_cb *wait_cb, + void *cb_data, + unsigned int timeout_sec); + #endif From e7939e07cb3abde73d326ee1ad36ec49723cc6bd Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 19 May 2021 13:45:55 -0400 Subject: [PATCH 053/110] fsmonitor: enhance existing comments Signed-off-by: Jeff Hostetler --- fsmonitor.c | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/fsmonitor.c b/fsmonitor.c index ab9bfc60b34e31..ec4c46407c5b53 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -301,9 +301,25 @@ void refresh_fsmonitor(struct index_state *istate) core_fsmonitor, query_success ? "success" : "failure"); } - /* a fsmonitor process can return '/' to indicate all entries are invalid */ + /* + * The response from FSMonitor (excluding the header token) is + * either: + * + * [a] a (possibly empty) list of NUL delimited relative + * pathnames of changed paths. This list can contain + * files and directories. Directories have a trailing + * slash. + * + * [b] a single '/' to indicate the provider had no + * information and that we should consider everything + * invalid. We call this a trivial response. + */ if (query_success && query_result.buf[bol] != '/') { - /* Mark all entries returned by the monitor as dirty */ + /* + * Mark all pathnames returned by the monitor as dirty. + * + * This updates both the cache-entries and the untracked-cache. + */ buf = query_result.buf; for (i = bol; i < query_result.len; i++) { if (buf[i] != '\0') @@ -318,11 +334,15 @@ void refresh_fsmonitor(struct index_state *istate) if (istate->untracked) istate->untracked->use_fsmonitor = 1; } else { - - /* We only want to run the post index changed hook if we've actually changed entries, so keep track - * if we actually changed entries or not */ + /* + * We received a trivial response, so invalidate everything. + * + * We only want to run the post index changed hook if + * we've actually changed entries, so keep track if we + * actually changed entries or not. + */ int is_cache_changed = 0; - /* Mark all entries invalid */ + for (i = 0; i < istate->cache_nr; i++) { if (istate->cache[i]->ce_flags & CE_FSMONITOR_VALID) { is_cache_changed = 1; @@ -330,7 +350,10 @@ void refresh_fsmonitor(struct index_state *istate) } } - /* If we're going to check every file, ensure we save the results */ + /* + * If we're going to check every file, ensure we save + * the results. + */ if (is_cache_changed) istate->cache_changed |= FSMONITOR_CHANGED; From c624a60d4b2986684f65ecb1f887f29715a8d311 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 12 May 2021 08:47:57 -0400 Subject: [PATCH 054/110] fsmonitor-ipc: create client routines for git-fsmonitor--daemon Create fsmonitor_ipc__*() client routines to spawn the built-in file system monitor daemon and send it an IPC request using the `Simple IPC` API. Stub in empty fsmonitor_ipc__*() functions for unsupported platforms. Signed-off-by: Jeff Hostetler --- Makefile | 1 + fsmonitor-ipc.c | 176 ++++++++++++++++++++++++++++++++++++++++++++++++ fsmonitor-ipc.h | 48 +++++++++++++ 3 files changed, 225 insertions(+) create mode 100644 fsmonitor-ipc.c create mode 100644 fsmonitor-ipc.h diff --git a/Makefile b/Makefile index 1ee317380c3eab..4f6920d39ebac0 100644 --- a/Makefile +++ b/Makefile @@ -903,6 +903,7 @@ LIB_OBJS += fetch-pack.o LIB_OBJS += fmt-merge-msg.o LIB_OBJS += fsck.o LIB_OBJS += fsmonitor.o +LIB_OBJS += fsmonitor-ipc.o LIB_OBJS += gettext.o LIB_OBJS += gpg-interface.o LIB_OBJS += graph.o diff --git a/fsmonitor-ipc.c b/fsmonitor-ipc.c new file mode 100644 index 00000000000000..ccc32d2a17ebe7 --- /dev/null +++ b/fsmonitor-ipc.c @@ -0,0 +1,176 @@ +#include "cache.h" +#include "fsmonitor.h" +#include "simple-ipc.h" +#include "fsmonitor-ipc.h" +#include "run-command.h" +#include "strbuf.h" +#include "trace2.h" + +#ifdef HAVE_FSMONITOR_DAEMON_BACKEND + +int fsmonitor_ipc__is_supported(void) +{ + return 1; +} + +GIT_PATH_FUNC(fsmonitor_ipc__get_path, "fsmonitor--daemon.ipc") + +enum ipc_active_state fsmonitor_ipc__get_state(void) +{ + return ipc_get_active_state(fsmonitor_ipc__get_path()); +} + +static int spawn_daemon(void) +{ + const char *args[] = { "fsmonitor--daemon", "start", NULL }; + + return run_command_v_opt_tr2(args, RUN_COMMAND_NO_STDIN | RUN_GIT_CMD, + "fsmonitor"); +} + +int fsmonitor_ipc__send_query(const char *since_token, + struct strbuf *answer) +{ + int ret = -1; + int tried_to_spawn = 0; + enum ipc_active_state state = IPC_STATE__OTHER_ERROR; + struct ipc_client_connection *connection = NULL; + struct ipc_client_connect_options options + = IPC_CLIENT_CONNECT_OPTIONS_INIT; + const char *tok = since_token ? since_token : ""; + size_t tok_len = since_token ? strlen(since_token) : 0; + + options.wait_if_busy = 1; + options.wait_if_not_found = 0; + + trace2_region_enter("fsm_client", "query", NULL); + trace2_data_string("fsm_client", NULL, "query/command", tok); + +try_again: + state = ipc_client_try_connect(fsmonitor_ipc__get_path(), &options, + &connection); + + switch (state) { + case IPC_STATE__LISTENING: + ret = ipc_client_send_command_to_connection( + connection, tok, tok_len, answer); + ipc_client_close_connection(connection); + + trace2_data_intmax("fsm_client", NULL, + "query/response-length", answer->len); + + if (fsmonitor_is_trivial_response(answer)) + trace2_data_intmax("fsm_client", NULL, + "query/trivial-response", 1); + + goto done; + + case IPC_STATE__NOT_LISTENING: + case IPC_STATE__PATH_NOT_FOUND: + if (tried_to_spawn) + goto done; + + tried_to_spawn++; + if (spawn_daemon()) + goto done; + + /* + * Try again, but this time give the daemon a chance to + * actually create the pipe/socket. + * + * Granted, the daemon just started so it can't possibly have + * any FS cached yet, so we'll always get a trivial answer. + * BUT the answer should include a new token that can serve + * as the basis for subsequent requests. + */ + options.wait_if_not_found = 1; + goto try_again; + + case IPC_STATE__INVALID_PATH: + ret = error(_("fsmonitor_ipc__send_query: invalid path '%s'"), + fsmonitor_ipc__get_path()); + goto done; + + case IPC_STATE__OTHER_ERROR: + default: + ret = error(_("fsmonitor_ipc__send_query: unspecified error on '%s'"), + fsmonitor_ipc__get_path()); + goto done; + } + +done: + trace2_region_leave("fsm_client", "query", NULL); + + return ret; +} + +int fsmonitor_ipc__send_command(const char *command, + struct strbuf *answer) +{ + struct ipc_client_connection *connection = NULL; + struct ipc_client_connect_options options + = IPC_CLIENT_CONNECT_OPTIONS_INIT; + int ret; + enum ipc_active_state state; + const char *c = command ? command : ""; + size_t c_len = command ? strlen(command) : 0; + + strbuf_reset(answer); + + options.wait_if_busy = 1; + options.wait_if_not_found = 0; + + state = ipc_client_try_connect(fsmonitor_ipc__get_path(), &options, + &connection); + if (state != IPC_STATE__LISTENING) { + die("fsmonitor--daemon is not running"); + return -1; + } + + ret = ipc_client_send_command_to_connection(connection, c, c_len, + answer); + ipc_client_close_connection(connection); + + if (ret == -1) { + die("could not send '%s' command to fsmonitor--daemon", c); + return -1; + } + + return 0; +} + +#else + +/* + * A trivial implementation of the fsmonitor_ipc__ API for unsupported + * platforms. + */ + +int fsmonitor_ipc__is_supported(void) +{ + return 0; +} + +const char *fsmonitor_ipc__get_path(void) +{ + return NULL; +} + +enum ipc_active_state fsmonitor_ipc__get_state(void) +{ + return IPC_STATE__OTHER_ERROR; +} + +int fsmonitor_ipc__send_query(const char *since_token, + struct strbuf *answer) +{ + return -1; +} + +int fsmonitor_ipc__send_command(const char *command, + struct strbuf *answer) +{ + return -1; +} + +#endif diff --git a/fsmonitor-ipc.h b/fsmonitor-ipc.h new file mode 100644 index 00000000000000..b6a7067c3af511 --- /dev/null +++ b/fsmonitor-ipc.h @@ -0,0 +1,48 @@ +#ifndef FSMONITOR_IPC_H +#define FSMONITOR_IPC_H + +#include "simple-ipc.h" + +/* + * Returns true if built-in file system monitor daemon is defined + * for this platform. + */ +int fsmonitor_ipc__is_supported(void); + +/* + * Returns the pathname to the IPC named pipe or Unix domain socket + * where a `git-fsmonitor--daemon` process will listen. This is a + * per-worktree value. + * + * Returns NULL if the daemon is not supported on this platform. + */ +const char *fsmonitor_ipc__get_path(void); + +/* + * Try to determine whether there is a `git-fsmonitor--daemon` process + * listening on the IPC pipe/socket. + */ +enum ipc_active_state fsmonitor_ipc__get_state(void); + +/* + * Connect to a `git-fsmonitor--daemon` process via simple-ipc + * and ask for the set of changed files since the given token. + * + * Spawn a daemon process in the background if necessary. + * + * Returns -1 on error; 0 on success. + */ +int fsmonitor_ipc__send_query(const char *since_token, + struct strbuf *answer); + +/* + * Connect to a `git-fsmonitor--daemon` process via simple-ipc and + * send a command verb. If no daemon is available, we DO NOT try to + * start one. + * + * Returns -1 on error; 0 on success. + */ +int fsmonitor_ipc__send_command(const char *command, + struct strbuf *answer); + +#endif /* FSMONITOR_IPC_H */ From c008c3bdd3f219151b27e501f6e4ad82f3cc5419 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 5 Mar 2021 21:20:26 +0100 Subject: [PATCH 055/110] fsmonitor: config settings are repository-specific Move FSMonitor config settings to a new `struct fsmonitor_settings` structure. Add a lazily-loaded pointer to `struct repo_settings`. Create `fsm_settings__get_*()` getters to lazily look up fsmonitor- related config settings. Get rid of the `core_fsmonitor` global variable, and add support for the new `core.useBuiltinFSMonitor` config setting. Move config code to lookup the existing `core.fsmonitor` value to `fsmonitor-settings.[ch]`. The `core_fsmonitor` global variable was used to store the pathname to the FSMonitor hook and it was used as a boolean to see if FSMonitor was enabled. This dual usage will lead to confusion when we add support for a builtin FSMonitor based on IPC, since the builtin FSMonitor doesn't need the hook pathname. Replace the boolean usage with an `enum fsmonitor_mode` to represent the state of FSMonitor. And only set the pathname when in HOOK mode. Signed-off-by: Jeff Hostetler --- Makefile | 1 + builtin/update-index.c | 19 +++++++-- cache.h | 1 - config.c | 14 ------ config.h | 1 - environment.c | 1 - fsmonitor-settings.c | 97 ++++++++++++++++++++++++++++++++++++++++++ fsmonitor-settings.h | 21 +++++++++ fsmonitor.c | 63 ++++++++++++++++----------- fsmonitor.h | 18 ++++++-- repo-settings.c | 2 + repository.h | 3 ++ t/README | 4 +- 13 files changed, 194 insertions(+), 51 deletions(-) create mode 100644 fsmonitor-settings.c create mode 100644 fsmonitor-settings.h diff --git a/Makefile b/Makefile index 4f6920d39ebac0..e595e9539d3e41 100644 --- a/Makefile +++ b/Makefile @@ -904,6 +904,7 @@ LIB_OBJS += fmt-merge-msg.o LIB_OBJS += fsck.o LIB_OBJS += fsmonitor.o LIB_OBJS += fsmonitor-ipc.o +LIB_OBJS += fsmonitor-settings.o LIB_OBJS += gettext.o LIB_OBJS += gpg-interface.o LIB_OBJS += graph.o diff --git a/builtin/update-index.c b/builtin/update-index.c index f1f16f2de526d9..25dae7360a601f 100644 --- a/builtin/update-index.c +++ b/builtin/update-index.c @@ -1216,14 +1216,25 @@ int cmd_update_index(int argc, const char **argv, const char *prefix) } if (fsmonitor > 0) { - if (git_config_get_fsmonitor() == 0) + enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); + + if (fsm_mode == FSMONITOR_MODE_DISABLED) { + warning(_("core.useBuiltinFSMonitor is unset; " + "set it if you really want to enable the " + "builtin fsmonitor")); warning(_("core.fsmonitor is unset; " - "set it if you really want to " - "enable fsmonitor")); + "set it if you really want to enable the " + "hook-based fsmonitor")); + } add_fsmonitor(&the_index); report(_("fsmonitor enabled")); } else if (!fsmonitor) { - if (git_config_get_fsmonitor() == 1) + enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); + if (fsm_mode == FSMONITOR_MODE_IPC) + warning(_("core.useBuiltinFSMonitor is set; " + "remove it if you really want to " + "disable fsmonitor")); + if (fsm_mode == FSMONITOR_MODE_HOOK) warning(_("core.fsmonitor is set; " "remove it if you really want to " "disable fsmonitor")); diff --git a/cache.h b/cache.h index f13a157d0372fd..eb5903e5fc4229 100644 --- a/cache.h +++ b/cache.h @@ -990,7 +990,6 @@ extern int core_preload_index; extern int precomposed_unicode; extern int protect_hfs; extern int protect_ntfs; -extern const char *core_fsmonitor; extern int core_apply_sparse_checkout; extern int core_sparse_checkout_cone; diff --git a/config.c b/config.c index f33abeab851542..19f1d719c81d57 100644 --- a/config.c +++ b/config.c @@ -2517,20 +2517,6 @@ int git_config_get_max_percent_split_change(void) return -1; /* default value */ } -int git_config_get_fsmonitor(void) -{ - if (git_config_get_pathname("core.fsmonitor", &core_fsmonitor)) - core_fsmonitor = getenv("GIT_TEST_FSMONITOR"); - - if (core_fsmonitor && !*core_fsmonitor) - core_fsmonitor = NULL; - - if (core_fsmonitor) - return 1; - - return 0; -} - int git_config_get_index_threads(int *dest) { int is_bool, val; diff --git a/config.h b/config.h index a2200f311156c4..110c426b082547 100644 --- a/config.h +++ b/config.h @@ -609,7 +609,6 @@ int git_config_get_index_threads(int *dest); int git_config_get_untracked_cache(void); int git_config_get_split_index(void); int git_config_get_max_percent_split_change(void); -int git_config_get_fsmonitor(void); /* This dies if the configured or default date is in the future */ int git_config_get_expiry(const char *key, const char **output); diff --git a/environment.c b/environment.c index d6b22ede7ea288..e6b66315284b57 100644 --- a/environment.c +++ b/environment.c @@ -84,7 +84,6 @@ int protect_hfs = PROTECT_HFS_DEFAULT; #define PROTECT_NTFS_DEFAULT 1 #endif int protect_ntfs = PROTECT_NTFS_DEFAULT; -const char *core_fsmonitor; /* * The character that begins a commented line in user-editable file diff --git a/fsmonitor-settings.c b/fsmonitor-settings.c new file mode 100644 index 00000000000000..2770266f5ee1ad --- /dev/null +++ b/fsmonitor-settings.c @@ -0,0 +1,97 @@ +#include "cache.h" +#include "config.h" +#include "repository.h" +#include "fsmonitor-settings.h" + +/* + * We keep this structure defintion private and have getters + * for all fields so that we can lazy load it as needed. + */ +struct fsmonitor_settings { + enum fsmonitor_mode mode; + char *hook_path; +}; + +void fsm_settings__set_ipc(struct repository *r) +{ + struct fsmonitor_settings *s = r->settings.fsmonitor; + + s->mode = FSMONITOR_MODE_IPC; +} + +void fsm_settings__set_hook(struct repository *r, const char *path) +{ + struct fsmonitor_settings *s = r->settings.fsmonitor; + + s->mode = FSMONITOR_MODE_HOOK; + s->hook_path = strdup(path); +} + +void fsm_settings__set_disabled(struct repository *r) +{ + struct fsmonitor_settings *s = r->settings.fsmonitor; + + s->mode = FSMONITOR_MODE_DISABLED; + FREE_AND_NULL(s->hook_path); +} + +static int check_for_ipc(struct repository *r) +{ + int value; + + if (!repo_config_get_bool(r, "core.usebuiltinfsmonitor", &value) && + value) { + fsm_settings__set_ipc(r); + return 1; + } + + return 0; +} + +static int check_for_hook(struct repository *r) +{ + const char *const_str; + + if (repo_config_get_pathname(r, "core.fsmonitor", &const_str)) + const_str = getenv("GIT_TEST_FSMONITOR"); + + if (const_str && *const_str) { + fsm_settings__set_hook(r, const_str); + return 1; + } + + return 0; +} + +static void lookup_fsmonitor_settings(struct repository *r) +{ + struct fsmonitor_settings *s; + + CALLOC_ARRAY(s, 1); + + r->settings.fsmonitor = s; + + if (check_for_ipc(r)) + return; + + if (check_for_hook(r)) + return; + + fsm_settings__set_disabled(r); +} + +enum fsmonitor_mode fsm_settings__get_mode(struct repository *r) +{ + if (!r->settings.fsmonitor) + lookup_fsmonitor_settings(r); + + return r->settings.fsmonitor->mode; +} + +const char *fsm_settings__get_hook_path(struct repository *r) +{ + if (!r->settings.fsmonitor) + lookup_fsmonitor_settings(r); + + return r->settings.fsmonitor->hook_path; +} diff --git a/fsmonitor-settings.h b/fsmonitor-settings.h new file mode 100644 index 00000000000000..50b2923461656c --- /dev/null +++ b/fsmonitor-settings.h @@ -0,0 +1,21 @@ +#ifndef FSMONITOR_SETTINGS_H +#define FSMONITOR_SETTINGS_H + +struct repository; + +enum fsmonitor_mode { + FSMONITOR_MODE_DISABLED = 0, + FSMONITOR_MODE_HOOK = 1, /* core.fsmonitor */ + FSMONITOR_MODE_IPC = 2, /* core.useBuiltinFSMonitor */ +}; + +void fsm_settings__set_ipc(struct repository *r); +void fsm_settings__set_hook(struct repository *r, const char *path); +void fsm_settings__set_disabled(struct repository *r); + +enum fsmonitor_mode fsm_settings__get_mode(struct repository *r); +const char *fsm_settings__get_hook_path(struct repository *r); + +struct fsmonitor_settings; + +#endif /* FSMONITOR_SETTINGS_H */ diff --git a/fsmonitor.c b/fsmonitor.c index ec4c46407c5b53..63174630c0e2b5 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -3,6 +3,7 @@ #include "dir.h" #include "ewah/ewok.h" #include "fsmonitor.h" +#include "fsmonitor-ipc.h" #include "run-command.h" #include "strbuf.h" @@ -148,15 +149,18 @@ void write_fsmonitor_extension(struct strbuf *sb, struct index_state *istate) /* * Call the query-fsmonitor hook passing the last update token of the saved results. */ -static int query_fsmonitor(int version, const char *last_update, struct strbuf *query_result) +static int query_fsmonitor_hook(struct repository *r, + int version, + const char *last_update, + struct strbuf *query_result) { struct child_process cp = CHILD_PROCESS_INIT; int result; - if (!core_fsmonitor) + if (fsm_settings__get_mode(r) != FSMONITOR_MODE_HOOK) return -1; - strvec_push(&cp.args, core_fsmonitor); + strvec_push(&cp.args, fsm_settings__get_hook_path(r)); strvec_pushf(&cp.args, "%d", version); strvec_pushf(&cp.args, "%s", last_update); cp.use_shell = 1; @@ -238,17 +242,28 @@ void refresh_fsmonitor(struct index_state *istate) struct strbuf last_update_token = STRBUF_INIT; char *buf; unsigned int i; + struct repository *r = istate->repo ? istate->repo : the_repository; + enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); - if (!core_fsmonitor || istate->fsmonitor_has_run_once) + if (fsm_mode <= FSMONITOR_MODE_DISABLED || + istate->fsmonitor_has_run_once) return; - hook_version = fsmonitor_hook_version(); - istate->fsmonitor_has_run_once = 1; trace_printf_key(&trace_fsmonitor, "refresh fsmonitor"); + + if (fsm_mode == FSMONITOR_MODE_IPC) { + /* TODO */ + return; + } + + assert(fsm_mode == FSMONITOR_MODE_HOOK); + + hook_version = fsmonitor_hook_version(); + /* - * This could be racy so save the date/time now and query_fsmonitor + * This could be racy so save the date/time now and query_fsmonitor_hook * should be inclusive to ensure we don't miss potential changes. */ last_update = getnanotime(); @@ -256,13 +271,14 @@ void refresh_fsmonitor(struct index_state *istate) strbuf_addf(&last_update_token, "%"PRIu64"", last_update); /* - * If we have a last update token, call query_fsmonitor for the set of + * If we have a last update token, call query_fsmonitor_hook for the set of * changes since that token, else assume everything is possibly dirty * and check it all. */ if (istate->fsmonitor_last_update) { if (hook_version == -1 || hook_version == HOOK_INTERFACE_VERSION2) { - query_success = !query_fsmonitor(HOOK_INTERFACE_VERSION2, + query_success = !query_fsmonitor_hook( + r, HOOK_INTERFACE_VERSION2, istate->fsmonitor_last_update, &query_result); if (query_success) { @@ -292,13 +308,17 @@ void refresh_fsmonitor(struct index_state *istate) } if (hook_version == HOOK_INTERFACE_VERSION1) { - query_success = !query_fsmonitor(HOOK_INTERFACE_VERSION1, + query_success = !query_fsmonitor_hook( + r, HOOK_INTERFACE_VERSION1, istate->fsmonitor_last_update, &query_result); } - trace_performance_since(last_update, "fsmonitor process '%s'", core_fsmonitor); - trace_printf_key(&trace_fsmonitor, "fsmonitor process '%s' returned %s", - core_fsmonitor, query_success ? "success" : "failure"); + trace_performance_since(last_update, "fsmonitor process '%s'", + fsm_settings__get_hook_path(r)); + trace_printf_key(&trace_fsmonitor, + "fsmonitor process '%s' returned %s", + fsm_settings__get_hook_path(r), + query_success ? "success" : "failure"); } /* @@ -434,7 +454,8 @@ void remove_fsmonitor(struct index_state *istate) void tweak_fsmonitor(struct index_state *istate) { unsigned int i; - int fsmonitor_enabled = git_config_get_fsmonitor(); + struct repository *r = istate->repo ? istate->repo : the_repository; + int fsmonitor_enabled = (fsm_settings__get_mode(r) > FSMONITOR_MODE_DISABLED); if (istate->fsmonitor_dirty) { if (fsmonitor_enabled) { @@ -454,16 +475,8 @@ void tweak_fsmonitor(struct index_state *istate) istate->fsmonitor_dirty = NULL; } - switch (fsmonitor_enabled) { - case -1: /* keep: do nothing */ - break; - case 0: /* false */ - remove_fsmonitor(istate); - break; - case 1: /* true */ + if (fsmonitor_enabled) add_fsmonitor(istate); - break; - default: /* unknown value: do nothing */ - break; - } + else + remove_fsmonitor(istate); } diff --git a/fsmonitor.h b/fsmonitor.h index f20d72631d76b4..f9201411aa74f4 100644 --- a/fsmonitor.h +++ b/fsmonitor.h @@ -3,6 +3,7 @@ #include "cache.h" #include "dir.h" +#include "fsmonitor-settings.h" extern struct trace_key trace_fsmonitor; @@ -57,7 +58,11 @@ int fsmonitor_is_trivial_response(const struct strbuf *query_result); */ static inline int is_fsmonitor_refreshed(const struct index_state *istate) { - return !core_fsmonitor || istate->fsmonitor_has_run_once; + struct repository *r = istate->repo ? istate->repo : the_repository; + enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); + + return fsm_mode <= FSMONITOR_MODE_DISABLED || + istate->fsmonitor_has_run_once; } /* @@ -67,7 +72,11 @@ static inline int is_fsmonitor_refreshed(const struct index_state *istate) */ static inline void mark_fsmonitor_valid(struct index_state *istate, struct cache_entry *ce) { - if (core_fsmonitor && !(ce->ce_flags & CE_FSMONITOR_VALID)) { + struct repository *r = istate->repo ? istate->repo : the_repository; + enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); + + if (fsm_mode > FSMONITOR_MODE_DISABLED && + !(ce->ce_flags & CE_FSMONITOR_VALID)) { istate->cache_changed = 1; ce->ce_flags |= CE_FSMONITOR_VALID; trace_printf_key(&trace_fsmonitor, "mark_fsmonitor_clean '%s'", ce->name); @@ -83,7 +92,10 @@ static inline void mark_fsmonitor_valid(struct index_state *istate, struct cache */ static inline void mark_fsmonitor_invalid(struct index_state *istate, struct cache_entry *ce) { - if (core_fsmonitor) { + struct repository *r = istate->repo ? istate->repo : the_repository; + enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); + + if (fsm_mode > FSMONITOR_MODE_DISABLED) { ce->ce_flags &= ~CE_FSMONITOR_VALID; untracked_cache_invalidate_path(istate, ce->name, 1); trace_printf_key(&trace_fsmonitor, "mark_fsmonitor_invalid '%s'", ce->name); diff --git a/repo-settings.c b/repo-settings.c index 0cfe8b787db26d..e69dad1e776772 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -26,6 +26,8 @@ void prepare_repo_settings(struct repository *r) UPDATE_DEFAULT_BOOL(r->settings.commit_graph_read_changed_paths, 1); UPDATE_DEFAULT_BOOL(r->settings.gc_write_commit_graph, 1); + r->settings.fsmonitor = NULL; /* lazy loaded */ + if (!repo_config_get_int(r, "index.version", &value)) r->settings.index_version = value; if (!repo_config_get_maybe_bool(r, "core.untrackedcache", &value)) { diff --git a/repository.h b/repository.h index 3740c93bc0fe27..fdc0a818b83c08 100644 --- a/repository.h +++ b/repository.h @@ -4,6 +4,7 @@ #include "path.h" struct config_set; +struct fsmonitor_settings; struct git_hash_algo; struct index_state; struct lock_file; @@ -35,6 +36,8 @@ struct repo_settings { int gc_write_commit_graph; int fetch_write_commit_graph; + struct fsmonitor_settings *fsmonitor; /* lazy loaded */ + int index_version; enum untracked_cache_setting core_untracked_cache; diff --git a/t/README b/t/README index 02ee9ce143fb5e..eba2be8fee0e2b 100644 --- a/t/README +++ b/t/README @@ -398,8 +398,8 @@ every 'git commit-graph write', as if the `--changed-paths` option was passed in. GIT_TEST_FSMONITOR=$PWD/t7519/fsmonitor-all exercises the fsmonitor -code path for utilizing a file system monitor to speed up detecting -new or changed files. +code path for utilizing a (hook based) file system monitor to speed up +detecting new or changed files. GIT_TEST_INDEX_VERSION= exercises the index read/write code path for the index version specified. Can be set to any valid version From 3587437d0c10c952bcfc9f04e2adcfc053d88b33 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 2 Aug 2019 19:17:25 +0200 Subject: [PATCH 056/110] fsmonitor: use IPC to query the builtin FSMonitor daemon Use simple IPC to directly communicate with the new builtin file system monitor daemon when `core.useBuiltinFSMonitor` is set. The `core.fsmonitor` setting has already been defined as a HOOK pathname. Historically, this has been set to a HOOK script that will talk with Watchman. For compatibility reasons, we do not want to overload that definition (and cause problems if users have multiple versions of Git installed). Signed-off-by: Johannes Schindelin Signed-off-by: Jeff Hostetler --- fsmonitor.c | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/fsmonitor.c b/fsmonitor.c index 63174630c0e2b5..695fb0ce4e7d75 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -254,8 +254,37 @@ void refresh_fsmonitor(struct index_state *istate) trace_printf_key(&trace_fsmonitor, "refresh fsmonitor"); if (fsm_mode == FSMONITOR_MODE_IPC) { - /* TODO */ - return; + query_success = !fsmonitor_ipc__send_query( + istate->fsmonitor_last_update ? + istate->fsmonitor_last_update : "builtin:fake", + &query_result); + if (query_success) { + /* + * The response contains a series of nul terminated + * strings. The first is the new token. + * + * Use `char *buf` as an interlude to trick the CI + * static analysis to let us use `strbuf_addstr()` + * here (and only copy the token) rather than + * `strbuf_addbuf()`. + */ + buf = query_result.buf; + strbuf_addstr(&last_update_token, buf); + bol = last_update_token.len + 1; + } else { + /* + * The builtin daemon is not available on this + * platform -OR- we failed to get a response. + * + * Generate a fake token (rather than a V1 + * timestamp) for the index extension. (If + * they switch back to the hook API, we don't + * want ambiguous state.) + */ + strbuf_addstr(&last_update_token, "builtin:fake"); + } + + goto apply_results; } assert(fsm_mode == FSMONITOR_MODE_HOOK); @@ -321,6 +350,7 @@ void refresh_fsmonitor(struct index_state *istate) query_success ? "success" : "failure"); } +apply_results: /* * The response from FSMonitor (excluding the header token) is * either: From 64e68477ef6f37fe89b688019bc25c5d4f877aec Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 16 Aug 2021 13:54:09 -0400 Subject: [PATCH 057/110] t/helper/simple-ipc: convert test-simple-ipc to use start_bg_command Convert test helper to use `start_bg_command()` when spawning a server daemon in the background rather than blocks of platform-specific code. Also, while here, remove _() translation around error messages since this is a test helper and not Git code. Signed-off-by: Jeff Hostetler --- t/helper/test-simple-ipc.c | 199 ++++++++----------------------------- 1 file changed, 43 insertions(+), 156 deletions(-) diff --git a/t/helper/test-simple-ipc.c b/t/helper/test-simple-ipc.c index 9134518075097e..28365ff85b69bb 100644 --- a/t/helper/test-simple-ipc.c +++ b/t/helper/test-simple-ipc.c @@ -9,6 +9,7 @@ #include "parse-options.h" #include "thread-utils.h" #include "strvec.h" +#include "run-command.h" #ifndef SUPPORTS_SIMPLE_IPC int cmd__simple_ipc(int argc, const char **argv) @@ -267,185 +268,71 @@ static int daemon__run_server(void) */ ret = ipc_server_run(cl_args.path, &opts, test_app_cb, (void*)&my_app_data); if (ret == -2) - error(_("socket/pipe already in use: '%s'"), cl_args.path); + error("socket/pipe already in use: '%s'", cl_args.path); else if (ret == -1) - error_errno(_("could not start server on: '%s'"), cl_args.path); + error_errno("could not start server on: '%s'", cl_args.path); return ret; } -#ifndef GIT_WINDOWS_NATIVE -/* - * This is adapted from `daemonize()`. Use `fork()` to directly create and - * run the daemon in a child process. - */ -static int spawn_server(pid_t *pid) -{ - struct ipc_server_opts opts = { - .nr_threads = cl_args.nr_threads, - }; +static start_bg_wait_cb bg_wait_cb; - *pid = fork(); - - switch (*pid) { - case 0: - if (setsid() == -1) - error_errno(_("setsid failed")); - close(0); - close(1); - close(2); - sanitize_stdfds(); +static int bg_wait_cb(const struct child_process *cp, void *cb_data) +{ + int s = ipc_get_active_state(cl_args.path); - return ipc_server_run(cl_args.path, &opts, test_app_cb, - (void*)&my_app_data); + switch (s) { + case IPC_STATE__LISTENING: + /* child is "ready" */ + return 0; - case -1: - return error_errno(_("could not spawn daemon in the background")); + case IPC_STATE__NOT_LISTENING: + case IPC_STATE__PATH_NOT_FOUND: + /* give child more time */ + return 1; default: - return 0; + case IPC_STATE__INVALID_PATH: + case IPC_STATE__OTHER_ERROR: + /* all the time in world won't help */ + return -1; } } -#else -/* - * Conceptually like `daemonize()` but different because Windows does not - * have `fork(2)`. Spawn a normal Windows child process but without the - * limitations of `start_command()` and `finish_command()`. - */ -static int spawn_server(pid_t *pid) -{ - char test_tool_exe[MAX_PATH]; - struct strvec args = STRVEC_INIT; - int in, out; - - GetModuleFileNameA(NULL, test_tool_exe, MAX_PATH); - - in = open("/dev/null", O_RDONLY); - out = open("/dev/null", O_WRONLY); - - strvec_push(&args, test_tool_exe); - strvec_push(&args, "simple-ipc"); - strvec_push(&args, "run-daemon"); - strvec_pushf(&args, "--name=%s", cl_args.path); - strvec_pushf(&args, "--threads=%d", cl_args.nr_threads); - - *pid = mingw_spawnvpe(args.v[0], args.v, NULL, NULL, in, out, out); - close(in); - close(out); - - strvec_clear(&args); - if (*pid < 0) - return error(_("could not spawn daemon in the background")); - - return 0; -} -#endif - -/* - * This is adapted from `wait_or_whine()`. Watch the child process and - * let it get started and begin listening for requests on the socket - * before reporting our success. - */ -static int wait_for_server_startup(pid_t pid_child) +static int daemon__start_server(void) { - int status; - pid_t pid_seen; - enum ipc_active_state s; - time_t time_limit, now; + struct child_process cp = CHILD_PROCESS_INIT; + enum start_bg_result sbgr; - time(&time_limit); - time_limit += cl_args.max_wait_sec; + strvec_push(&cp.args, "test-tool"); + strvec_push(&cp.args, "simple-ipc"); + strvec_push(&cp.args, "run-daemon"); + strvec_pushf(&cp.args, "--name=%s", cl_args.path); + strvec_pushf(&cp.args, "--threads=%d", cl_args.nr_threads); - for (;;) { - pid_seen = waitpid(pid_child, &status, WNOHANG); + cp.no_stdin = 1; + cp.no_stdout = 1; + cp.no_stderr = 1; - if (pid_seen == -1) - return error_errno(_("waitpid failed")); + sbgr = start_bg_command(&cp, bg_wait_cb, NULL, cl_args.max_wait_sec); - else if (pid_seen == 0) { - /* - * The child is still running (this should be - * the normal case). Try to connect to it on - * the socket and see if it is ready for - * business. - * - * If there is another daemon already running, - * our child will fail to start (possibly - * after a timeout on the lock), but we don't - * care (who responds) if the socket is live. - */ - s = ipc_get_active_state(cl_args.path); - if (s == IPC_STATE__LISTENING) - return 0; - - time(&now); - if (now > time_limit) - return error(_("daemon not online yet")); - - continue; - } + switch (sbgr) { + case SBGR_READY: + return 0; - else if (pid_seen == pid_child) { - /* - * The new child daemon process shutdown while - * it was starting up, so it is not listening - * on the socket. - * - * Try to ping the socket in the odd chance - * that another daemon started (or was already - * running) while our child was starting. - * - * Again, we don't care who services the socket. - */ - s = ipc_get_active_state(cl_args.path); - if (s == IPC_STATE__LISTENING) - return 0; + default: + case SBGR_ERROR: + case SBGR_CB_ERROR: + return error("daemon failed to start"); - /* - * We don't care about the WEXITSTATUS() nor - * any of the WIF*(status) values because - * `cmd__simple_ipc()` does the `!!result` - * trick on all function return values. - * - * So it is sufficient to just report the - * early shutdown as an error. - */ - return error(_("daemon failed to start")); - } + case SBGR_TIMEOUT: + return error("daemon not online yet"); - else - return error(_("waitpid is confused")); + case SBGR_DIED: + return error("daemon terminated"); } } -/* - * This process will start a simple-ipc server in a background process and - * wait for it to become ready. This is like `daemonize()` but gives us - * more control and better error reporting (and makes it easier to write - * unit tests). - */ -static int daemon__start_server(void) -{ - pid_t pid_child; - int ret; - - /* - * Run the actual daemon in a background process. - */ - ret = spawn_server(&pid_child); - if (pid_child <= 0) - return ret; - - /* - * Let the parent wait for the child process to get started - * and begin listening for requests on the socket. - */ - ret = wait_for_server_startup(pid_child); - - return ret; -} - /* * This process will run a quick probe to see if a simple-ipc server * is active on this path. @@ -548,7 +435,7 @@ static int client__stop_server(void) time(&now); if (now > time_limit) - return error(_("daemon has not shutdown yet")); + return error("daemon has not shutdown yet"); } } From 8d563f0824a05ea7ce13c92d2a02bc8415bcdcf3 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 11 May 2021 17:21:21 -0400 Subject: [PATCH 058/110] fsmonitor: update fsmonitor config documentation Update references to `core.fsmonitor` and `core.fsmonitorHookVersion` and pointers to `Watchman` to mention the new `core.useBuiltinFSMonitor` value. Signed-off-by: Jeff Hostetler --- Documentation/config/core.txt | 56 ++++++++++++++++++++++-------- Documentation/git-update-index.txt | 27 +++++++------- Documentation/githooks.txt | 3 +- 3 files changed, 59 insertions(+), 27 deletions(-) diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index eeb226d1b20784..323f966a09e6fd 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -62,22 +62,50 @@ core.protectNTFS:: Defaults to `true` on Windows, and `false` elsewhere. core.fsmonitor:: - If set, the value of this variable is used as a command which - will identify all files that may have changed since the - requested date/time. This information is used to speed up git by - avoiding unnecessary processing of files that have not changed. - See the "fsmonitor-watchman" section of linkgit:githooks[5]. + If set, this variable contains the pathname of the "fsmonitor" + hook command. ++ +This hook command is used to identify all files that may have changed +since the requested date/time. This information is used to speed up +git by avoiding unnecessary scanning of files that have not changed. ++ +See the "fsmonitor-watchman" section of linkgit:githooks[5]. ++ +Note: The value of this config setting is ignored if the +built-in file system monitor is enabled (see `core.useBuiltinFSMonitor`). core.fsmonitorHookVersion:: - Sets the version of hook that is to be used when calling fsmonitor. - There are currently versions 1 and 2. When this is not set, - version 2 will be tried first and if it fails then version 1 - will be tried. Version 1 uses a timestamp as input to determine - which files have changes since that time but some monitors - like watchman have race conditions when used with a timestamp. - Version 2 uses an opaque string so that the monitor can return - something that can be used to determine what files have changed - without race conditions. + Sets the protocol version to be used when invoking the + "fsmonitor" hook. ++ +There are currently versions 1 and 2. When this is not set, +version 2 will be tried first and if it fails then version 1 +will be tried. Version 1 uses a timestamp as input to determine +which files have changes since that time but some monitors +like Watchman have race conditions when used with a timestamp. +Version 2 uses an opaque string so that the monitor can return +something that can be used to determine what files have changed +without race conditions. ++ +Note: The value of this config setting is ignored if the +built-in file system monitor is enabled (see `core.useBuiltinFSMonitor`). + +core.useBuiltinFSMonitor:: + If set to true, enable the built-in file system monitor + daemon for this working directory (linkgit:git-fsmonitor--daemon[1]). ++ +Like hook-based file system monitors, the built-in file system monitor +can speed up Git commands that need to refresh the Git index +(e.g. `git status`) in a working directory with many files. The +built-in monitor eliminates the need to install and maintain an +external third-party tool. ++ +The built-in file system monitor is currently available only on a +limited set of supported platforms. Currently, this includes Windows +and MacOS. ++ +Note: if this config setting is set to `true`, the values of +`core.fsmonitor` and `core.fsmonitorHookVersion` are ignored. core.trustctime:: If false, the ctime differences between the index and the diff --git a/Documentation/git-update-index.txt b/Documentation/git-update-index.txt index 2853f168d97685..c7c31b3fcf9c66 100644 --- a/Documentation/git-update-index.txt +++ b/Documentation/git-update-index.txt @@ -498,7 +498,9 @@ FILE SYSTEM MONITOR This feature is intended to speed up git operations for repos that have large working directories. -It enables git to work together with a file system monitor (see the +It enables git to work together with a file system monitor (see +linkgit:git-fsmonitor--daemon[1] +and the "fsmonitor-watchman" section of linkgit:githooks[5]) that can inform it as to what files have been modified. This enables git to avoid having to lstat() every file to find modified files. @@ -508,17 +510,18 @@ performance by avoiding the cost of scanning the entire working directory looking for new files. If you want to enable (or disable) this feature, it is easier to use -the `core.fsmonitor` configuration variable (see -linkgit:git-config[1]) than using the `--fsmonitor` option to -`git update-index` in each repository, especially if you want to do so -across all repositories you use, because you can set the configuration -variable in your `$HOME/.gitconfig` just once and have it affect all -repositories you touch. - -When the `core.fsmonitor` configuration variable is changed, the -file system monitor is added to or removed from the index the next time -a command reads the index. When `--[no-]fsmonitor` are used, the file -system monitor is immediately added to or removed from the index. +the `core.fsmonitor` or `core.useBuiltinFSMonitor` configuration +variable (see linkgit:git-config[1]) than using the `--fsmonitor` +option to `git update-index` in each repository, especially if you +want to do so across all repositories you use, because you can set the +configuration variable in your `$HOME/.gitconfig` just once and have +it affect all repositories you touch. + +When the `core.fsmonitor` or `core.useBuiltinFSMonitor` configuration +variable is changed, the file system monitor is added to or removed +from the index the next time a command reads the index. When +`--[no-]fsmonitor` are used, the file system monitor is immediately +added to or removed from the index. CONFIGURATION ------------- diff --git a/Documentation/githooks.txt b/Documentation/githooks.txt index b51959ff9418fd..b7d5e926f7b042 100644 --- a/Documentation/githooks.txt +++ b/Documentation/githooks.txt @@ -593,7 +593,8 @@ fsmonitor-watchman This hook is invoked when the configuration option `core.fsmonitor` is set to `.git/hooks/fsmonitor-watchman` or `.git/hooks/fsmonitor-watchmanv2` -depending on the version of the hook to use. +depending on the version of the hook to use, unless overridden via +`core.useBuiltinFSMonitor` (see linkgit:git-config[1]). Version 1 takes two arguments, a version (1) and the time in elapsed nanoseconds since midnight, January 1, 1970. From 0f795e40f67ae1cd563856faa5142409f6bf3d1f Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 11 May 2021 17:20:18 -0400 Subject: [PATCH 059/110] fsmonitor--daemon: man page Create a manual page describing the `git fsmonitor--daemon` feature. Signed-off-by: Jeff Hostetler --- Documentation/git-fsmonitor--daemon.txt | 75 +++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 Documentation/git-fsmonitor--daemon.txt diff --git a/Documentation/git-fsmonitor--daemon.txt b/Documentation/git-fsmonitor--daemon.txt new file mode 100644 index 00000000000000..154e7684daae07 --- /dev/null +++ b/Documentation/git-fsmonitor--daemon.txt @@ -0,0 +1,75 @@ +git-fsmonitor--daemon(1) +======================== + +NAME +---- +git-fsmonitor--daemon - A Built-in File System Monitor + +SYNOPSIS +-------- +[verse] +'git fsmonitor--daemon' start +'git fsmonitor--daemon' run +'git fsmonitor--daemon' stop +'git fsmonitor--daemon' status + +DESCRIPTION +----------- + +A daemon to watch the working directory for file and directory +changes using platform-specific file system notification facilities. + +This daemon communicates directly with commands like `git status` +using the link:technical/api-simple-ipc.html[simple IPC] interface +instead of the slower linkgit:githooks[5] interface. + +This daemon is built into Git so that no third-party tools are +required. + +OPTIONS +------- + +start:: + Starts a daemon in the background. + +run:: + Runs a daemon in the foreground. + +stop:: + Stops the daemon running in the current working + directory, if present. + +status:: + Exits with zero status if a daemon is watching the + current working directory. + +REMARKS +------- + +This daemon is a long running process used to watch a single working +directory and maintain a list of the recently changed files and +directories. Performance of commands such as `git status` can be +increased if they just ask for a summary of changes to the working +directory and can avoid scanning the disk. + +When `core.useBuiltinFSMonitor` is set to `true` (see +linkgit:git-config[1]) commands, such as `git status`, will ask the +daemon for changes and automatically start it (if necessary). + +For more information see the "File System Monitor" section in +linkgit:git-update-index[1]. + +CAVEATS +------- + +The fsmonitor daemon does not currently know about submodules and does +not know to filter out file system events that happen within a +submodule. If fsmonitor daemon is watching a super repo and a file is +modified within the working directory of a submodule, it will report +the change (as happening against the super repo). However, the client +will properly ignore these extra events, so performance may be affected +but it will not cause an incorrect result. + +GIT +--- +Part of the linkgit:git[1] suite From 19e42f8f8ed3c6b609696e9e0cc09d2055aa93cf Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 16 Dec 2020 10:24:30 -0500 Subject: [PATCH 060/110] fsmonitor--daemon: add a built-in fsmonitor daemon Create a built-in file system monitoring daemon that can be used by the existing `fsmonitor` feature (protocol API and index extension) to improve the performance of various Git commands, such as `status`. The `fsmonitor--daemon` feature builds upon the `Simple IPC` API and provides an alternative to hook access to existing fsmonitors such as `watchman`. This commit merely adds the new command without any functionality. Co-authored-by: Johannes Schindelin Signed-off-by: Jeff Hostetler --- .gitignore | 1 + Makefile | 1 + builtin.h | 1 + builtin/fsmonitor--daemon.c | 53 +++++++++++++++++++++++++++++++++++++ git.c | 1 + 5 files changed, 57 insertions(+) create mode 100644 builtin/fsmonitor--daemon.c diff --git a/.gitignore b/.gitignore index 311841f9bed577..4baba472aa8261 100644 --- a/.gitignore +++ b/.gitignore @@ -72,6 +72,7 @@ /git-format-patch /git-fsck /git-fsck-objects +/git-fsmonitor--daemon /git-gc /git-get-tar-commit-id /git-grep diff --git a/Makefile b/Makefile index e595e9539d3e41..f62752c1cccdce 100644 --- a/Makefile +++ b/Makefile @@ -1109,6 +1109,7 @@ BUILTIN_OBJS += builtin/fmt-merge-msg.o BUILTIN_OBJS += builtin/for-each-ref.o BUILTIN_OBJS += builtin/for-each-repo.o BUILTIN_OBJS += builtin/fsck.o +BUILTIN_OBJS += builtin/fsmonitor--daemon.o BUILTIN_OBJS += builtin/gc.o BUILTIN_OBJS += builtin/get-tar-commit-id.o BUILTIN_OBJS += builtin/grep.o diff --git a/builtin.h b/builtin.h index 16ecd5586f0bee..2470d1cd3a267a 100644 --- a/builtin.h +++ b/builtin.h @@ -159,6 +159,7 @@ int cmd_for_each_ref(int argc, const char **argv, const char *prefix); int cmd_for_each_repo(int argc, const char **argv, const char *prefix); int cmd_format_patch(int argc, const char **argv, const char *prefix); int cmd_fsck(int argc, const char **argv, const char *prefix); +int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix); int cmd_gc(int argc, const char **argv, const char *prefix); int cmd_get_tar_commit_id(int argc, const char **argv, const char *prefix); int cmd_grep(int argc, const char **argv, const char *prefix); diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c new file mode 100644 index 00000000000000..df2bad531118c8 --- /dev/null +++ b/builtin/fsmonitor--daemon.c @@ -0,0 +1,53 @@ +#include "builtin.h" +#include "config.h" +#include "parse-options.h" +#include "fsmonitor.h" +#include "fsmonitor-ipc.h" +#include "simple-ipc.h" +#include "khash.h" + +static const char * const builtin_fsmonitor__daemon_usage[] = { + NULL +}; + +#ifdef HAVE_FSMONITOR_DAEMON_BACKEND + +int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) +{ + const char *subcmd; + + struct option options[] = { + OPT_END() + }; + + if (argc < 2) + usage_with_options(builtin_fsmonitor__daemon_usage, options); + + if (argc == 2 && !strcmp(argv[1], "-h")) + usage_with_options(builtin_fsmonitor__daemon_usage, options); + + git_config(git_default_config, NULL); + + subcmd = argv[1]; + argv--; + argc++; + + argc = parse_options(argc, argv, prefix, options, + builtin_fsmonitor__daemon_usage, 0); + + die(_("Unhandled subcommand '%s'"), subcmd); +} + +#else +int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) +{ + struct option options[] = { + OPT_END() + }; + + if (argc == 2 && !strcmp(argv[1], "-h")) + usage_with_options(builtin_fsmonitor__daemon_usage, options); + + die(_("fsmonitor--daemon not supported on this platform")); +} +#endif diff --git a/git.c b/git.c index 18bed9a99647aa..c6160f4a88612c 100644 --- a/git.c +++ b/git.c @@ -533,6 +533,7 @@ static struct cmd_struct commands[] = { { "format-patch", cmd_format_patch, RUN_SETUP }, { "fsck", cmd_fsck, RUN_SETUP }, { "fsck-objects", cmd_fsck, RUN_SETUP }, + { "fsmonitor--daemon", cmd_fsmonitor__daemon, RUN_SETUP }, { "gc", cmd_gc, RUN_SETUP }, { "get-tar-commit-id", cmd_get_tar_commit_id, NO_PARSEOPT }, { "grep", cmd_grep, RUN_SETUP_GENTLY }, From 0e97154174fb8fb137da3d2aca8a33ab60592cea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlo=20Marcelo=20Arenas=20Bel=C3=B3n?= Date: Tue, 17 Aug 2021 03:27:10 -0700 Subject: [PATCH 061/110] builtin/fsmonitor--daemon: use parse-options API fully MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --help and -h are already handled internally so just parse_options() do the parsing and extract the command from the remaining options. as a side effect, avoid setting a variable argc to a value that was never used. Signed-off-by: Carlo Marcelo Arenas Belón Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index df2bad531118c8..f04987933796bf 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -20,20 +20,13 @@ int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) OPT_END() }; - if (argc < 2) - usage_with_options(builtin_fsmonitor__daemon_usage, options); - - if (argc == 2 && !strcmp(argv[1], "-h")) - usage_with_options(builtin_fsmonitor__daemon_usage, options); - git_config(git_default_config, NULL); - subcmd = argv[1]; - argv--; - argc++; - argc = parse_options(argc, argv, prefix, options, builtin_fsmonitor__daemon_usage, 0); + if (argc != 1) + usage_with_options(builtin_fsmonitor__daemon_usage, options); + subcmd = argv[0]; die(_("Unhandled subcommand '%s'"), subcmd); } From 94ec0e57e32754a512d7d1e69422d8091af21870 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 16 Dec 2020 11:02:46 -0500 Subject: [PATCH 062/110] fsmonitor--daemon: implement 'stop' and 'status' commands Implement `stop` and `status` client commands to control and query the status of a `fsmonitor--daemon` server process (and implicitly start a server process if necessary). Later commits will implement the actual server and monitor the file system. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 51 +++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index f04987933796bf..5e3178b8bddbc6 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -7,10 +7,55 @@ #include "khash.h" static const char * const builtin_fsmonitor__daemon_usage[] = { + N_("git fsmonitor--daemon stop"), + N_("git fsmonitor--daemon status"), NULL }; #ifdef HAVE_FSMONITOR_DAEMON_BACKEND +/* + * Acting as a CLIENT. + * + * Send a "quit" command to the `git-fsmonitor--daemon` (if running) + * and wait for it to shutdown. + */ +static int do_as_client__send_stop(void) +{ + struct strbuf answer = STRBUF_INIT; + int ret; + + ret = fsmonitor_ipc__send_command("quit", &answer); + + /* The quit command does not return any response data. */ + strbuf_release(&answer); + + if (ret) + return ret; + + trace2_region_enter("fsm_client", "polling-for-daemon-exit", NULL); + while (fsmonitor_ipc__get_state() == IPC_STATE__LISTENING) + sleep_millisec(50); + trace2_region_leave("fsm_client", "polling-for-daemon-exit", NULL); + + return 0; +} + +static int do_as_client__status(void) +{ + enum ipc_active_state state = fsmonitor_ipc__get_state(); + + switch (state) { + case IPC_STATE__LISTENING: + printf(_("fsmonitor-daemon is watching '%s'\n"), + the_repository->worktree); + return 0; + + default: + printf(_("fsmonitor-daemon is not watching '%s'\n"), + the_repository->worktree); + return 1; + } +} int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) { @@ -28,6 +73,12 @@ int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) usage_with_options(builtin_fsmonitor__daemon_usage, options); subcmd = argv[0]; + if (!strcmp(subcmd, "stop")) + return !!do_as_client__send_stop(); + + if (!strcmp(subcmd, "status")) + return !!do_as_client__status(); + die(_("Unhandled subcommand '%s'"), subcmd); } From 3aac461ddf99f66ea51f1b2dc5c7ab3d7cff9976 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 17 Dec 2020 10:29:17 -0500 Subject: [PATCH 063/110] compat/fsmonitor/fsm-listen-win32: stub in backend for Windows Stub in empty filesystem listener backend for fsmonitor--daemon on Windows. Signed-off-by: Jeff Hostetler --- Makefile | 13 ++++++++ compat/fsmonitor/fsm-listen-win32.c | 21 +++++++++++++ compat/fsmonitor/fsm-listen.h | 49 +++++++++++++++++++++++++++++ config.mak.uname | 10 ++++++ contrib/buildsystems/CMakeLists.txt | 7 +++++ repo-settings.c | 1 + 6 files changed, 101 insertions(+) create mode 100644 compat/fsmonitor/fsm-listen-win32.c create mode 100644 compat/fsmonitor/fsm-listen.h diff --git a/Makefile b/Makefile index f62752c1cccdce..bd7c807c4f7a3f 100644 --- a/Makefile +++ b/Makefile @@ -471,6 +471,11 @@ all:: # directory, and the JSON compilation database 'compile_commands.json' will be # created at the root of the repository. # +# If your platform supports a built-in fsmonitor backend, set +# FSMONITOR_DAEMON_BACKEND to the "" of the corresponding +# `compat/fsmonitor/fsm-listen-.c` that implements the +# `fsm_listen__*()` routines. +# # Define DEVELOPER to enable more compiler warnings. Compiler version # and family are auto detected, but could be overridden by defining # COMPILER_FEATURES (see config.mak.dev). You can still set @@ -1940,6 +1945,11 @@ ifdef NEED_ACCESS_ROOT_HANDLER COMPAT_OBJS += compat/access.o endif +ifdef FSMONITOR_DAEMON_BACKEND + COMPAT_CFLAGS += -DHAVE_FSMONITOR_DAEMON_BACKEND + COMPAT_OBJS += compat/fsmonitor/fsm-listen-$(FSMONITOR_DAEMON_BACKEND).o +endif + ifeq ($(TCLTK_PATH),) NO_TCLTK = NoThanks endif @@ -2821,6 +2831,9 @@ GIT-BUILD-OPTIONS: FORCE @echo PAGER_ENV=\''$(subst ','\'',$(subst ','\'',$(PAGER_ENV)))'\' >>$@+ @echo DC_SHA1=\''$(subst ','\'',$(subst ','\'',$(DC_SHA1)))'\' >>$@+ @echo X=\'$(X)\' >>$@+ +ifdef FSMONITOR_DAEMON_BACKEND + @echo FSMONITOR_DAEMON_BACKEND=\''$(subst ','\'',$(subst ','\'',$(FSMONITOR_DAEMON_BACKEND)))'\' >>$@+ +endif ifdef TEST_OUTPUT_DIRECTORY @echo TEST_OUTPUT_DIRECTORY=\''$(subst ','\'',$(subst ','\'',$(TEST_OUTPUT_DIRECTORY)))'\' >>$@+ endif diff --git a/compat/fsmonitor/fsm-listen-win32.c b/compat/fsmonitor/fsm-listen-win32.c new file mode 100644 index 00000000000000..916cbea254f8cf --- /dev/null +++ b/compat/fsmonitor/fsm-listen-win32.c @@ -0,0 +1,21 @@ +#include "cache.h" +#include "config.h" +#include "fsmonitor.h" +#include "fsm-listen.h" + +void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) +{ +} + +void fsm_listen__loop(struct fsmonitor_daemon_state *state) +{ +} + +int fsm_listen__ctor(struct fsmonitor_daemon_state *state) +{ + return -1; +} + +void fsm_listen__dtor(struct fsmonitor_daemon_state *state) +{ +} diff --git a/compat/fsmonitor/fsm-listen.h b/compat/fsmonitor/fsm-listen.h new file mode 100644 index 00000000000000..f0539349baf848 --- /dev/null +++ b/compat/fsmonitor/fsm-listen.h @@ -0,0 +1,49 @@ +#ifndef FSM_LISTEN_H +#define FSM_LISTEN_H + +/* This needs to be implemented by each backend */ + +#ifdef HAVE_FSMONITOR_DAEMON_BACKEND + +struct fsmonitor_daemon_state; + +/* + * Initialize platform-specific data for the fsmonitor listener thread. + * This will be called from the main thread PRIOR to staring the + * fsmonitor_fs_listener thread. + * + * Returns 0 if successful. + * Returns -1 otherwise. + */ +int fsm_listen__ctor(struct fsmonitor_daemon_state *state); + +/* + * Cleanup platform-specific data for the fsmonitor listener thread. + * This will be called from the main thread AFTER joining the listener. + */ +void fsm_listen__dtor(struct fsmonitor_daemon_state *state); + +/* + * The main body of the platform-specific event loop to watch for + * filesystem events. This will run in the fsmonitor_fs_listen thread. + * + * It should call `ipc_server_stop_async()` if the listener thread + * prematurely terminates (because of a filesystem error or if it + * detects that the .git directory has been deleted). (It should NOT + * do so if the listener thread receives a normal shutdown signal from + * the IPC layer.) + * + * It should set `state->error_code` to -1 if the daemon should exit + * with an error. + */ +void fsm_listen__loop(struct fsmonitor_daemon_state *state); + +/* + * Gently request that the fsmonitor listener thread shutdown. + * It does not wait for it to stop. The caller should do a JOIN + * to wait for it. + */ +void fsm_listen__stop_async(struct fsmonitor_daemon_state *state); + +#endif /* HAVE_FSMONITOR_DAEMON_BACKEND */ +#endif /* FSM_LISTEN_H */ diff --git a/config.mak.uname b/config.mak.uname index d92da1fff13781..56643c89b5c3b7 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -420,6 +420,11 @@ ifeq ($(uname_S),Windows) # so we don't need this: # # SNPRINTF_RETURNS_BOGUS = YesPlease + + # The builtin FSMonitor requires Named Pipes and Threads on Windows. + # These are always available, so we do not have to conditionally + # support it. + FSMONITOR_DAEMON_BACKEND = win32 NO_SVN_TESTS = YesPlease RUNTIME_PREFIX = YesPlease HAVE_WPGMPTR = YesWeDo @@ -607,6 +612,11 @@ ifneq (,$(findstring MINGW,$(uname_S))) NO_STRTOUMAX = YesPlease NO_MKDTEMP = YesPlease NO_SVN_TESTS = YesPlease + + # The builtin FSMonitor requires Named Pipes and Threads on Windows. + # These are always available, so we do not have to conditionally + # support it. + FSMONITOR_DAEMON_BACKEND = win32 RUNTIME_PREFIX = YesPlease HAVE_WPGMPTR = YesWeDo NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index a4ffbacfa4b83f..77926588b80701 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -292,6 +292,13 @@ else() endif() endif() +if(SUPPORTS_SIMPLE_IPC) + if(CMAKE_SYSTEM_NAME STREQUAL "Windows") + add_compile_definitions(HAVE_FSMONITOR_DAEMON_BACKEND) + list(APPEND compat_SOURCES compat/fsmonitor/fsm-listen-win32.c) + endif() +endif() + set(EXE_EXTENSION ${CMAKE_EXECUTABLE_SUFFIX}) #header checks diff --git a/repo-settings.c b/repo-settings.c index e69dad1e776772..08c60a927473e2 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -2,6 +2,7 @@ #include "config.h" #include "repository.h" #include "midx.h" +#include "compat/fsmonitor/fsm-listen.h" #define UPDATE_DEFAULT_BOOL(s,v) do { if (s == -1) { s = v; } } while(0) From c569886157007e2320cc7e930820026397341b27 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 17 Dec 2020 10:41:07 -0500 Subject: [PATCH 064/110] compat/fsmonitor/fsm-listen-darwin: stub in backend for Darwin Stub in empty implementation of fsmonitor--daemon backend for Darwin (aka MacOS). Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-listen-darwin.c | 20 ++++++++++++++++++++ config.mak.uname | 10 ++++++++++ contrib/buildsystems/CMakeLists.txt | 3 +++ 3 files changed, 33 insertions(+) create mode 100644 compat/fsmonitor/fsm-listen-darwin.c diff --git a/compat/fsmonitor/fsm-listen-darwin.c b/compat/fsmonitor/fsm-listen-darwin.c new file mode 100644 index 00000000000000..c84e3344ab99b4 --- /dev/null +++ b/compat/fsmonitor/fsm-listen-darwin.c @@ -0,0 +1,20 @@ +#include "cache.h" +#include "fsmonitor.h" +#include "fsm-listen.h" + +int fsm_listen__ctor(struct fsmonitor_daemon_state *state) +{ + return -1; +} + +void fsm_listen__dtor(struct fsmonitor_daemon_state *state) +{ +} + +void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) +{ +} + +void fsm_listen__loop(struct fsmonitor_daemon_state *state) +{ +} diff --git a/config.mak.uname b/config.mak.uname index 56643c89b5c3b7..a15cf715cc1c7d 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -147,6 +147,16 @@ ifeq ($(uname_S),Darwin) MSGFMT = /usr/local/opt/gettext/bin/msgfmt endif endif + + # The builtin FSMonitor on MacOS builds upon Simple-IPC. Both require + # Unix domain sockets and PThreads. + ifndef NO_PTHREADS + ifndef NO_UNIX_SOCKETS + FSMONITOR_DAEMON_BACKEND = darwin + endif + endif + + BASIC_LDFLAGS += -framework CoreServices endif ifeq ($(uname_S),SunOS) NEEDS_SOCKET = YesPlease diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 77926588b80701..58bc3383de323e 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -296,6 +296,9 @@ if(SUPPORTS_SIMPLE_IPC) if(CMAKE_SYSTEM_NAME STREQUAL "Windows") add_compile_definitions(HAVE_FSMONITOR_DAEMON_BACKEND) list(APPEND compat_SOURCES compat/fsmonitor/fsm-listen-win32.c) + elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + add_compile_definitions(HAVE_FSMONITOR_DAEMON_BACKEND) + list(APPEND compat_SOURCES compat/fsmonitor/fsm-listen-darwin.c) endif() endif() From 2685c53a7c73d0b216240fd17129455377125a0a Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 29 Jun 2021 09:50:16 -0400 Subject: [PATCH 065/110] fsmonitor--daemon: implement 'run' command Implement `run` command to try to begin listening for file system events. This version defines the thread structure with a single fsmonitor_fs_listen thread to watch for file system events and a simple IPC thread pool to watch for connection from Git clients over a well-known named pipe or Unix domain socket. This commit does not actually do anything yet because the platform backends are still just stubs. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 213 +++++++++++++++++++++++++++++++++++- fsmonitor--daemon.h | 34 ++++++ 2 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 fsmonitor--daemon.h diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 5e3178b8bddbc6..b5ebd1eca33dde 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -3,16 +3,39 @@ #include "parse-options.h" #include "fsmonitor.h" #include "fsmonitor-ipc.h" +#include "compat/fsmonitor/fsm-listen.h" +#include "fsmonitor--daemon.h" #include "simple-ipc.h" #include "khash.h" static const char * const builtin_fsmonitor__daemon_usage[] = { + N_("git fsmonitor--daemon run []"), N_("git fsmonitor--daemon stop"), N_("git fsmonitor--daemon status"), NULL }; #ifdef HAVE_FSMONITOR_DAEMON_BACKEND +/* + * Global state loaded from config. + */ +#define FSMONITOR__IPC_THREADS "fsmonitor.ipcthreads" +static int fsmonitor__ipc_threads = 8; + +static int fsmonitor_config(const char *var, const char *value, void *cb) +{ + if (!strcmp(var, FSMONITOR__IPC_THREADS)) { + int i = git_config_int(var, value); + if (i < 1) + return error(_("value of '%s' out of range: %d"), + FSMONITOR__IPC_THREADS, i); + fsmonitor__ipc_threads = i; + return 0; + } + + return git_default_config(var, value, cb); +} + /* * Acting as a CLIENT. * @@ -57,15 +80,196 @@ static int do_as_client__status(void) } } +static ipc_server_application_cb handle_client; + +static int handle_client(void *data, + const char *command, size_t command_len, + ipc_server_reply_cb *reply, + struct ipc_server_reply_data *reply_data) +{ + /* struct fsmonitor_daemon_state *state = data; */ + int result; + + /* + * The Simple IPC API now supports {char*, len} arguments, but + * FSMonitor always uses proper null-terminated strings, so + * we can ignore the command_len argument. (Trust, but verify.) + */ + if (command_len != strlen(command)) + BUG("FSMonitor assumes text messages"); + + trace2_region_enter("fsmonitor", "handle_client", the_repository); + trace2_data_string("fsmonitor", the_repository, "request", command); + + result = 0; /* TODO Do something here. */ + + trace2_region_leave("fsmonitor", "handle_client", the_repository); + + return result; +} + +static void *fsm_listen__thread_proc(void *_state) +{ + struct fsmonitor_daemon_state *state = _state; + + trace2_thread_start("fsm-listen"); + + trace_printf_key(&trace_fsmonitor, "Watching: worktree '%s'", + state->path_worktree_watch.buf); + if (state->nr_paths_watching > 1) + trace_printf_key(&trace_fsmonitor, "Watching: gitdir '%s'", + state->path_gitdir_watch.buf); + + fsm_listen__loop(state); + + trace2_thread_exit(); + return NULL; +} + +static int fsmonitor_run_daemon_1(struct fsmonitor_daemon_state *state) +{ + struct ipc_server_opts ipc_opts = { + .nr_threads = fsmonitor__ipc_threads, + + /* + * We know that there are no other active threads yet, + * so we can let the IPC layer temporarily chdir() if + * it needs to when creating the server side of the + * Unix domain socket. + */ + .uds_disallow_chdir = 0 + }; + + /* + * Start the IPC thread pool before the we've started the file + * system event listener thread so that we have the IPC handle + * before we need it. + */ + if (ipc_server_run_async(&state->ipc_server_data, + fsmonitor_ipc__get_path(), &ipc_opts, + handle_client, state)) + return error_errno( + _("could not start IPC thread pool on '%s'"), + fsmonitor_ipc__get_path()); + + /* + * Start the fsmonitor listener thread to collect filesystem + * events. + */ + if (pthread_create(&state->listener_thread, NULL, + fsm_listen__thread_proc, state) < 0) { + ipc_server_stop_async(state->ipc_server_data); + ipc_server_await(state->ipc_server_data); + + return error(_("could not start fsmonitor listener thread")); + } + + /* + * The daemon is now fully functional in background threads. + * Wait for the IPC thread pool to shutdown (whether by client + * request or from filesystem activity). + */ + ipc_server_await(state->ipc_server_data); + + /* + * The fsmonitor listener thread may have received a shutdown + * event from the IPC thread pool, but it doesn't hurt to tell + * it again. And wait for it to shutdown. + */ + fsm_listen__stop_async(state); + pthread_join(state->listener_thread, NULL); + + return state->error_code; +} + +static int fsmonitor_run_daemon(void) +{ + struct fsmonitor_daemon_state state; + int err; + + memset(&state, 0, sizeof(state)); + + pthread_mutex_init(&state.main_lock, NULL); + state.error_code = 0; + state.current_token_data = NULL; + + /* Prepare to (recursively) watch the directory. */ + strbuf_init(&state.path_worktree_watch, 0); + strbuf_addstr(&state.path_worktree_watch, absolute_path(get_git_work_tree())); + state.nr_paths_watching = 1; + + /* + * We create and delete cookie files somewhere inside the .git + * directory to help us keep sync with the file system. If + * ".git" is not a directory, then is not inside the + * cone of , so set up a second watch to watch + * the so that we get events for the cookie files. + */ + strbuf_init(&state.path_gitdir_watch, 0); + strbuf_addbuf(&state.path_gitdir_watch, &state.path_worktree_watch); + strbuf_addstr(&state.path_gitdir_watch, "/.git"); + if (!is_directory(state.path_gitdir_watch.buf)) { + strbuf_reset(&state.path_gitdir_watch); + strbuf_addstr(&state.path_gitdir_watch, absolute_path(get_git_dir())); + state.nr_paths_watching = 2; + } + + /* + * Confirm that we can create platform-specific resources for the + * filesystem listener before we bother starting all the threads. + */ + if (fsm_listen__ctor(&state)) { + err = error(_("could not initialize listener thread")); + goto done; + } + + err = fsmonitor_run_daemon_1(&state); + +done: + pthread_mutex_destroy(&state.main_lock); + fsm_listen__dtor(&state); + + ipc_server_free(state.ipc_server_data); + + strbuf_release(&state.path_worktree_watch); + strbuf_release(&state.path_gitdir_watch); + + return err; +} + +static int try_to_run_foreground_daemon(void) +{ + /* + * Technically, we don't need to probe for an existing daemon + * process, since we could just call `fsmonitor_run_daemon()` + * and let it fail if the pipe/socket is busy. + * + * However, this method gives us a nicer error message for a + * common error case. + */ + if (fsmonitor_ipc__get_state() == IPC_STATE__LISTENING) + die("fsmonitor--daemon is already running '%s'", + the_repository->worktree); + + printf(_("running fsmonitor-daemon in '%s'\n"), + the_repository->worktree); + fflush(stdout); + + return !!fsmonitor_run_daemon(); +} + int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) { const char *subcmd; struct option options[] = { + OPT_INTEGER(0, "ipc-threads", + &fsmonitor__ipc_threads, + N_("use ipc worker threads")), OPT_END() }; - git_config(git_default_config, NULL); + git_config(fsmonitor_config, NULL); argc = parse_options(argc, argv, prefix, options, builtin_fsmonitor__daemon_usage, 0); @@ -73,6 +277,13 @@ int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) usage_with_options(builtin_fsmonitor__daemon_usage, options); subcmd = argv[0]; + if (fsmonitor__ipc_threads < 1) + die(_("invalid 'ipc-threads' value (%d)"), + fsmonitor__ipc_threads); + + if (!strcmp(subcmd, "run")) + return !!try_to_run_foreground_daemon(); + if (!strcmp(subcmd, "stop")) return !!do_as_client__send_stop(); diff --git a/fsmonitor--daemon.h b/fsmonitor--daemon.h new file mode 100644 index 00000000000000..3009c1a83de7f6 --- /dev/null +++ b/fsmonitor--daemon.h @@ -0,0 +1,34 @@ +#ifndef FSMONITOR_DAEMON_H +#define FSMONITOR_DAEMON_H + +#ifdef HAVE_FSMONITOR_DAEMON_BACKEND + +#include "cache.h" +#include "dir.h" +#include "run-command.h" +#include "simple-ipc.h" +#include "thread-utils.h" + +struct fsmonitor_batch; +struct fsmonitor_token_data; + +struct fsmonitor_daemon_backend_data; /* opaque platform-specific data */ + +struct fsmonitor_daemon_state { + pthread_t listener_thread; + pthread_mutex_t main_lock; + + struct strbuf path_worktree_watch; + struct strbuf path_gitdir_watch; + int nr_paths_watching; + + struct fsmonitor_token_data *current_token_data; + + int error_code; + struct fsmonitor_daemon_backend_data *backend_data; + + struct ipc_server_data *ipc_server_data; +}; + +#endif /* HAVE_FSMONITOR_DAEMON_BACKEND */ +#endif /* FSMONITOR_DAEMON_H */ From 34d228cf966400e1e6d06915e7a0775a5754659b Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 29 Jun 2021 10:28:15 -0400 Subject: [PATCH 066/110] fsmonitor--daemon: implement 'start' command Implement 'git fsmonitor--daemon start' command. This command starts an instance of 'git fsmonitor--daemon run' in the background using the new 'start_bg_command()' function. We avoid the fork-and-call technique on Unix systems in favor of a fork-and-exec technique. This gives us more uniform Trace2 child-* events. It also makes our usage more consistent with Windows usage. On Windows, teach 'git fsmonitor--daemon run' to optionally call 'FreeConsole()' to release handles to the inherited Win32 console (despite being passed invalid handles for stdin/out/err). Without this, command prompts and powershell terminal windows could hang in "exit" until the last background child process exited or released their Win32 console handle. (This was not seen with git-bash shells because they don't have a Win32 console attached to them.) Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 107 +++++++++++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 2 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index b5ebd1eca33dde..8988440b7c1108 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -9,6 +9,7 @@ #include "khash.h" static const char * const builtin_fsmonitor__daemon_usage[] = { + N_("git fsmonitor--daemon start []"), N_("git fsmonitor--daemon run []"), N_("git fsmonitor--daemon stop"), N_("git fsmonitor--daemon status"), @@ -22,6 +23,9 @@ static const char * const builtin_fsmonitor__daemon_usage[] = { #define FSMONITOR__IPC_THREADS "fsmonitor.ipcthreads" static int fsmonitor__ipc_threads = 8; +#define FSMONITOR__START_TIMEOUT "fsmonitor.starttimeout" +static int fsmonitor__start_timeout_sec = 60; + static int fsmonitor_config(const char *var, const char *value, void *cb) { if (!strcmp(var, FSMONITOR__IPC_THREADS)) { @@ -33,6 +37,15 @@ static int fsmonitor_config(const char *var, const char *value, void *cb) return 0; } + if (!strcmp(var, FSMONITOR__START_TIMEOUT)) { + int i = git_config_int(var, value); + if (i < 0) + return error(_("value of '%s' out of range: %d"), + FSMONITOR__START_TIMEOUT, i); + fsmonitor__start_timeout_sec = i; + return 0; + } + return git_default_config(var, value, cb); } @@ -237,7 +250,7 @@ static int fsmonitor_run_daemon(void) return err; } -static int try_to_run_foreground_daemon(void) +static int try_to_run_foreground_daemon(int free_console) { /* * Technically, we don't need to probe for an existing daemon @@ -255,17 +268,104 @@ static int try_to_run_foreground_daemon(void) the_repository->worktree); fflush(stdout); +#ifdef GIT_WINDOWS_NATIVE + if (free_console) + FreeConsole(); +#endif + return !!fsmonitor_run_daemon(); } +static start_bg_wait_cb bg_wait_cb; + +static int bg_wait_cb(const struct child_process *cp, void *cb_data) +{ + enum ipc_active_state s = fsmonitor_ipc__get_state(); + + switch (s) { + case IPC_STATE__LISTENING: + /* child is "ready" */ + return 0; + + case IPC_STATE__NOT_LISTENING: + case IPC_STATE__PATH_NOT_FOUND: + /* give child more time */ + return 1; + + default: + case IPC_STATE__INVALID_PATH: + case IPC_STATE__OTHER_ERROR: + /* all the time in world won't help */ + return -1; + } +} + +static int try_to_start_background_daemon(void) +{ + struct child_process cp = CHILD_PROCESS_INIT; + enum start_bg_result sbgr; + + /* + * Before we try to create a background daemon process, see + * if a daemon process is already listening. This makes it + * easier for us to report an already-listening error to the + * console, since our spawn/daemon can only report the success + * of creating the background process (and not whether it + * immediately exited). + */ + if (fsmonitor_ipc__get_state() == IPC_STATE__LISTENING) + die("fsmonitor--daemon is already running '%s'", + the_repository->worktree); + + printf(_("starting fsmonitor-daemon in '%s'\n"), + the_repository->worktree); + fflush(stdout); + + cp.git_cmd = 1; + + strvec_push(&cp.args, "fsmonitor--daemon"); + strvec_push(&cp.args, "run"); + strvec_push(&cp.args, "--free-console"); + strvec_pushf(&cp.args, "--ipc-threads=%d", fsmonitor__ipc_threads); + + cp.no_stdin = 1; + cp.no_stdout = 1; + cp.no_stderr = 1; + + sbgr = start_bg_command(&cp, bg_wait_cb, NULL, + fsmonitor__start_timeout_sec); + + switch (sbgr) { + case SBGR_READY: + return 0; + + default: + case SBGR_ERROR: + case SBGR_CB_ERROR: + return error("daemon failed to start"); + + case SBGR_TIMEOUT: + return error("daemon not online yet"); + + case SBGR_DIED: + return error("daemon terminated"); + } +} + int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) { const char *subcmd; + int free_console = 0; struct option options[] = { + OPT_BOOL(0, "free-console", &free_console, N_("free console")), OPT_INTEGER(0, "ipc-threads", &fsmonitor__ipc_threads, N_("use ipc worker threads")), + OPT_INTEGER(0, "start-timeout", + &fsmonitor__start_timeout_sec, + N_("Max seconds to wait for background daemon startup")), + OPT_END() }; @@ -281,8 +381,11 @@ int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) die(_("invalid 'ipc-threads' value (%d)"), fsmonitor__ipc_threads); + if (!strcmp(subcmd, "start")) + return !!try_to_start_background_daemon(); + if (!strcmp(subcmd, "run")) - return !!try_to_run_foreground_daemon(); + return !!try_to_run_foreground_daemon(free_console); if (!strcmp(subcmd, "stop")) return !!do_as_client__send_stop(); From 64971b4d86f2dd87e04fb7ce3a32fcd8ffee2c54 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 17 Dec 2020 12:37:15 -0500 Subject: [PATCH 067/110] fsmonitor--daemon: add pathname classification Teach fsmonitor--daemon to classify relative and absolute pathnames and decide how they should be handled. This will be used by the platform-specific backend to respond to each filesystem event. When we register for filesystem notifications on a directory, we get events for everything (recursively) in the directory. We want to report to clients changes to tracked and untracked paths within the working directory. We do not want to report changes within the .git directory, for example. This classification will be used in a later commit by the different backends to classify paths as events are received. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 81 +++++++++++++++++++++++++++++++++++++ fsmonitor--daemon.h | 61 ++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 8988440b7c1108..1618f1aa27fc5e 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -121,6 +121,87 @@ static int handle_client(void *data, return result; } +#define FSMONITOR_COOKIE_PREFIX ".fsmonitor-daemon-" + +enum fsmonitor_path_type fsmonitor_classify_path_workdir_relative( + const char *rel) +{ + if (fspathncmp(rel, ".git", 4)) + return IS_WORKDIR_PATH; + rel += 4; + + if (!*rel) + return IS_DOT_GIT; + if (*rel != '/') + return IS_WORKDIR_PATH; /* e.g. .gitignore */ + rel++; + + if (!fspathncmp(rel, FSMONITOR_COOKIE_PREFIX, + strlen(FSMONITOR_COOKIE_PREFIX))) + return IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX; + + return IS_INSIDE_DOT_GIT; +} + +enum fsmonitor_path_type fsmonitor_classify_path_gitdir_relative( + const char *rel) +{ + if (!fspathncmp(rel, FSMONITOR_COOKIE_PREFIX, + strlen(FSMONITOR_COOKIE_PREFIX))) + return IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX; + + return IS_INSIDE_GITDIR; +} + +static enum fsmonitor_path_type try_classify_workdir_abs_path( + struct fsmonitor_daemon_state *state, + const char *path) +{ + const char *rel; + + if (fspathncmp(path, state->path_worktree_watch.buf, + state->path_worktree_watch.len)) + return IS_OUTSIDE_CONE; + + rel = path + state->path_worktree_watch.len; + + if (!*rel) + return IS_WORKDIR_PATH; /* it is the root dir exactly */ + if (*rel != '/') + return IS_OUTSIDE_CONE; + rel++; + + return fsmonitor_classify_path_workdir_relative(rel); +} + +enum fsmonitor_path_type fsmonitor_classify_path_absolute( + struct fsmonitor_daemon_state *state, + const char *path) +{ + const char *rel; + enum fsmonitor_path_type t; + + t = try_classify_workdir_abs_path(state, path); + if (state->nr_paths_watching == 1) + return t; + if (t != IS_OUTSIDE_CONE) + return t; + + if (fspathncmp(path, state->path_gitdir_watch.buf, + state->path_gitdir_watch.len)) + return IS_OUTSIDE_CONE; + + rel = path + state->path_gitdir_watch.len; + + if (!*rel) + return IS_GITDIR; /* it is the exactly */ + if (*rel != '/') + return IS_OUTSIDE_CONE; + rel++; + + return fsmonitor_classify_path_gitdir_relative(rel); +} + static void *fsm_listen__thread_proc(void *_state) { struct fsmonitor_daemon_state *state = _state; diff --git a/fsmonitor--daemon.h b/fsmonitor--daemon.h index 3009c1a83de7f6..7bbb3a27a1ce1d 100644 --- a/fsmonitor--daemon.h +++ b/fsmonitor--daemon.h @@ -30,5 +30,66 @@ struct fsmonitor_daemon_state { struct ipc_server_data *ipc_server_data; }; +/* + * Pathname classifications. + * + * The daemon classifies the pathnames that it receives from file + * system notification events into the following categories and uses + * that to decide whether clients are told about them. (And to watch + * for file system synchronization events.) + * + * The client should only care about paths within the working + * directory proper (inside the working directory and not ".git" nor + * inside of ".git/"). That is, the client has read the index and is + * asking for a list of any paths in the working directory that have + * been modified since the last token. The client does not care about + * file system changes within the .git directory (such as new loose + * objects or packfiles). So the client will only receive paths that + * are classified as IS_WORKDIR_PATH. + * + * The daemon uses the IS_DOT_GIT and IS_GITDIR internally to mean the + * exact ".git" directory or GITDIR. If the daemon receives a delete + * event for either of these directories, it will automatically + * shutdown, for example. + * + * Note that the daemon DOES NOT explicitly watch nor special case the + * ".git/index" file. The daemon does not read the index and does not + * have any internal index-relative state. The daemon only collects + * the set of modified paths within the working directory. + */ +enum fsmonitor_path_type { + IS_WORKDIR_PATH = 0, + + IS_DOT_GIT, + IS_INSIDE_DOT_GIT, + IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX, + + IS_GITDIR, + IS_INSIDE_GITDIR, + IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX, + + IS_OUTSIDE_CONE, +}; + +/* + * Classify a pathname relative to the root of the working directory. + */ +enum fsmonitor_path_type fsmonitor_classify_path_workdir_relative( + const char *relative_path); + +/* + * Classify a pathname relative to a that is external to the + * worktree directory. + */ +enum fsmonitor_path_type fsmonitor_classify_path_gitdir_relative( + const char *relative_path); + +/* + * Classify an absolute pathname received from a filesystem event. + */ +enum fsmonitor_path_type fsmonitor_classify_path_absolute( + struct fsmonitor_daemon_state *state, + const char *path); + #endif /* HAVE_FSMONITOR_DAEMON_BACKEND */ #endif /* FSMONITOR_DAEMON_H */ From 556722075f5883dc894a262a1ceb90242b373cbb Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 17 Dec 2020 12:58:14 -0500 Subject: [PATCH 068/110] fsmonitor--daemon: define token-ids Teach fsmonitor--daemon to create token-ids and define the overall token naming scheme. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 116 +++++++++++++++++++++++++++++++++++- 1 file changed, 115 insertions(+), 1 deletion(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 1618f1aa27fc5e..2997c2cba8ce2d 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -93,6 +93,120 @@ static int do_as_client__status(void) } } +/* + * Requests to and from a FSMonitor Protocol V2 provider use an opaque + * "token" as a virtual timestamp. Clients can request a summary of all + * created/deleted/modified files relative to a token. In the response, + * clients receive a new token for the next (relative) request. + * + * + * Token Format + * ============ + * + * The contents of the token are private and provider-specific. + * + * For the built-in fsmonitor--daemon, we define a token as follows: + * + * "builtin" ":" ":" + * + * The "builtin" prefix is used as a namespace to avoid conflicts + * with other providers (such as Watchman). + * + * The is an arbitrary OPAQUE string, such as a GUID, + * UUID, or {timestamp,pid}. It is used to group all filesystem + * events that happened while the daemon was monitoring (and in-sync + * with the filesystem). + * + * Unlike FSMonitor Protocol V1, it is not defined as a timestamp + * and does not define less-than/greater-than relationships. + * (There are too many race conditions to rely on file system + * event timestamps.) + * + * The is a simple integer incremented whenever the + * daemon needs to make its state public. For example, if 1000 file + * system events come in, but no clients have requested the data, + * the daemon can continue to accumulate file changes in the same + * bin and does not need to advance the sequence number. However, + * as soon as a client does arrive, the daemon needs to start a new + * bin and increment the sequence number. + * + * The sequence number serves as the boundary between 2 sets + * of bins -- the older ones that the client has already seen + * and the newer ones that it hasn't. + * + * When a new is created, the is reset to + * zero. + * + * + * About Token Ids + * =============== + * + * A new token_id is created: + * + * [1] each time the daemon is started. + * + * [2] any time that the daemon must re-sync with the filesystem + * (such as when the kernel drops or we miss events on a very + * active volume). + * + * [3] in response to a client "flush" command (for dropped event + * testing). + * + * When a new token_id is created, the daemon is free to discard all + * cached filesystem events associated with any previous token_ids. + * Events associated with a non-current token_id will never be sent + * to a client. A token_id change implicitly means that the daemon + * has gap in its event history. + * + * Therefore, clients that present a token with a stale (non-current) + * token_id will always be given a trivial response. + */ +struct fsmonitor_token_data { + struct strbuf token_id; + struct fsmonitor_batch *batch_head; + struct fsmonitor_batch *batch_tail; + uint64_t client_ref_count; +}; + +static struct fsmonitor_token_data *fsmonitor_new_token_data(void) +{ + static int test_env_value = -1; + static uint64_t flush_count = 0; + struct fsmonitor_token_data *token; + + CALLOC_ARRAY(token, 1); + + strbuf_init(&token->token_id, 0); + token->batch_head = NULL; + token->batch_tail = NULL; + token->client_ref_count = 0; + + if (test_env_value < 0) + test_env_value = git_env_bool("GIT_TEST_FSMONITOR_TOKEN", 0); + + if (!test_env_value) { + struct timeval tv; + struct tm tm; + time_t secs; + + gettimeofday(&tv, NULL); + secs = tv.tv_sec; + gmtime_r(&secs, &tm); + + strbuf_addf(&token->token_id, + "%"PRIu64".%d.%4d%02d%02dT%02d%02d%02d.%06ldZ", + flush_count++, + getpid(), + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec, + (long)tv.tv_usec); + } else { + strbuf_addf(&token->token_id, "test_%08x", test_env_value++); + } + + return token; +} + static ipc_server_application_cb handle_client; static int handle_client(void *data, @@ -285,7 +399,7 @@ static int fsmonitor_run_daemon(void) pthread_mutex_init(&state.main_lock, NULL); state.error_code = 0; - state.current_token_data = NULL; + state.current_token_data = fsmonitor_new_token_data(); /* Prepare to (recursively) watch the directory. */ strbuf_init(&state.path_worktree_watch, 0); From f54b4a4d15dc95850753e84f1b84b0f12529f167 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 17 Dec 2020 13:17:33 -0500 Subject: [PATCH 069/110] fsmonitor--daemon: create token-based changed path cache Teach fsmonitor--daemon to build a list of changed paths and associate them with a token-id. This will be used by the platform-specific backends to accumulate changed paths in response to filesystem events. The platform-specific file system listener thread receives file system events containing one or more changed pathnames (with whatever bucketing or grouping that is convenient for the file system). These paths are accumulated (without locking) by the file system layer into a `fsmonitor_batch`. When the file system layer has drained the kernel event queue, it will "publish" them to our token queue and make them visible to concurrent client worker threads. The token layer is free to combine and/or de-dup paths within these batches for efficient presentation to clients. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 230 +++++++++++++++++++++++++++++++++++- fsmonitor--daemon.h | 40 +++++++ 2 files changed, 268 insertions(+), 2 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 2997c2cba8ce2d..34603f23053f61 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -168,17 +168,27 @@ struct fsmonitor_token_data { uint64_t client_ref_count; }; +struct fsmonitor_batch { + struct fsmonitor_batch *next; + uint64_t batch_seq_nr; + const char **interned_paths; + size_t nr, alloc; + time_t pinned_time; +}; + static struct fsmonitor_token_data *fsmonitor_new_token_data(void) { static int test_env_value = -1; static uint64_t flush_count = 0; struct fsmonitor_token_data *token; + struct fsmonitor_batch *batch; CALLOC_ARRAY(token, 1); + batch = fsmonitor_batch__new(); strbuf_init(&token->token_id, 0); - token->batch_head = NULL; - token->batch_tail = NULL; + token->batch_head = batch; + token->batch_tail = batch; token->client_ref_count = 0; if (test_env_value < 0) @@ -204,9 +214,143 @@ static struct fsmonitor_token_data *fsmonitor_new_token_data(void) strbuf_addf(&token->token_id, "test_%08x", test_env_value++); } + /* + * We created a new and are starting a new series + * of tokens with a zero . + * + * Since clients cannot guess our new (non test) + * they will always receive a trivial response (because of the + * mismatch on the ). The trivial response will + * tell them our new so that subsequent requests + * will be relative to our new series. (And when sending that + * response, we pin the current head of the batch list.) + * + * Even if the client correctly guesses the , their + * request of "builtin::0" asks for all changes MORE + * RECENT than batch/bin 0. + * + * This implies that it is a waste to accumulate paths in the + * initial batch/bin (because they will never be transmitted). + * + * So the daemon could be running for days and watching the + * file system, but doesn't need to actually accumulate any + * paths UNTIL we need to set a reference point for a later + * relative request. + * + * However, it is very useful for testing to always have a + * reference point set. Pin batch 0 to force early file system + * events to accumulate. + */ + if (test_env_value) + batch->pinned_time = time(NULL); + return token; } +struct fsmonitor_batch *fsmonitor_batch__new(void) +{ + struct fsmonitor_batch *batch; + + CALLOC_ARRAY(batch, 1); + + return batch; +} + +void fsmonitor_batch__free_list(struct fsmonitor_batch *batch) +{ + while (batch) { + struct fsmonitor_batch *next = batch->next; + + /* + * The actual strings within the array of this batch + * are interned, so we don't own them. We only own + * the array. + */ + free(batch->interned_paths); + free(batch); + + batch = next; + } +} + +void fsmonitor_batch__add_path(struct fsmonitor_batch *batch, + const char *path) +{ + const char *interned_path = strintern(path); + + trace_printf_key(&trace_fsmonitor, "event: %s", interned_path); + + ALLOC_GROW(batch->interned_paths, batch->nr + 1, batch->alloc); + batch->interned_paths[batch->nr++] = interned_path; +} + +static void fsmonitor_batch__combine(struct fsmonitor_batch *batch_dest, + const struct fsmonitor_batch *batch_src) +{ + size_t k; + + ALLOC_GROW(batch_dest->interned_paths, + batch_dest->nr + batch_src->nr + 1, + batch_dest->alloc); + + for (k = 0; k < batch_src->nr; k++) + batch_dest->interned_paths[batch_dest->nr++] = + batch_src->interned_paths[k]; +} + +static void fsmonitor_free_token_data(struct fsmonitor_token_data *token) +{ + if (!token) + return; + + assert(token->client_ref_count == 0); + + strbuf_release(&token->token_id); + + fsmonitor_batch__free_list(token->batch_head); + + free(token); +} + +/* + * Flush all of our cached data about the filesystem. Call this if we + * lose sync with the filesystem and miss some notification events. + * + * [1] If we are missing events, then we no longer have a complete + * history of the directory (relative to our current start token). + * We should create a new token and start fresh (as if we just + * booted up). + * + * If there are no concurrent threads readering the current token data + * series, we can free it now. Otherwise, let the last reader free + * it. + * + * Either way, the old token data series is no longer associated with + * our state data. + */ +static void with_lock__do_force_resync(struct fsmonitor_daemon_state *state) +{ + /* assert current thread holding state->main_lock */ + + struct fsmonitor_token_data *free_me = NULL; + struct fsmonitor_token_data *new_one = NULL; + + new_one = fsmonitor_new_token_data(); + + if (state->current_token_data->client_ref_count == 0) + free_me = state->current_token_data; + state->current_token_data = new_one; + + fsmonitor_free_token_data(free_me); +} + +void fsmonitor_force_resync(struct fsmonitor_daemon_state *state) +{ + pthread_mutex_lock(&state->main_lock); + with_lock__do_force_resync(state); + pthread_mutex_unlock(&state->main_lock); +} + static ipc_server_application_cb handle_client; static int handle_client(void *data, @@ -316,6 +460,81 @@ enum fsmonitor_path_type fsmonitor_classify_path_absolute( return fsmonitor_classify_path_gitdir_relative(rel); } +/* + * We try to combine small batches at the front of the batch-list to avoid + * having a long list. This hopefully makes it a little easier when we want + * to truncate and maintain the list. However, we don't want the paths array + * to just keep growing and growing with realloc, so we insert an arbitrary + * limit. + */ +#define MY_COMBINE_LIMIT (1024) + +void fsmonitor_publish(struct fsmonitor_daemon_state *state, + struct fsmonitor_batch *batch, + const struct string_list *cookie_names) +{ + if (!batch && !cookie_names->nr) + return; + + pthread_mutex_lock(&state->main_lock); + + if (batch) { + struct fsmonitor_batch *head; + + head = state->current_token_data->batch_head; + if (!head) { + BUG("token does not have batch"); + } else if (head->pinned_time) { + /* + * We cannot alter the current batch list + * because: + * + * [a] it is being transmitted to at least one + * client and the handle_client() thread has a + * ref-count, but not a lock on the batch list + * starting with this item. + * + * [b] it has been transmitted in the past to + * at least one client such that future + * requests are relative to this head batch. + * + * So, we can only prepend a new batch onto + * the front of the list. + */ + batch->batch_seq_nr = head->batch_seq_nr + 1; + batch->next = head; + state->current_token_data->batch_head = batch; + } else if (!head->batch_seq_nr) { + /* + * Batch 0 is unpinned. See the note in + * `fsmonitor_new_token_data()` about why we + * don't need to accumulate these paths. + */ + fsmonitor_batch__free_list(batch); + } else if (head->nr + batch->nr > MY_COMBINE_LIMIT) { + /* + * The head batch in the list has never been + * transmitted to a client, but folding the + * contents of the new batch onto it would + * exceed our arbitrary limit, so just prepend + * the new batch onto the list. + */ + batch->batch_seq_nr = head->batch_seq_nr + 1; + batch->next = head; + state->current_token_data->batch_head = batch; + } else { + /* + * We are free to add the paths in the given + * batch onto the end of the current head batch. + */ + fsmonitor_batch__combine(head, batch); + fsmonitor_batch__free_list(batch); + } + } + + pthread_mutex_unlock(&state->main_lock); +} + static void *fsm_listen__thread_proc(void *_state) { struct fsmonitor_daemon_state *state = _state; @@ -330,6 +549,13 @@ static void *fsm_listen__thread_proc(void *_state) fsm_listen__loop(state); + pthread_mutex_lock(&state->main_lock); + if (state->current_token_data && + state->current_token_data->client_ref_count == 0) + fsmonitor_free_token_data(state->current_token_data); + state->current_token_data = NULL; + pthread_mutex_unlock(&state->main_lock); + trace2_thread_exit(); return NULL; } diff --git a/fsmonitor--daemon.h b/fsmonitor--daemon.h index 7bbb3a27a1ce1d..20a815d80f8081 100644 --- a/fsmonitor--daemon.h +++ b/fsmonitor--daemon.h @@ -12,6 +12,27 @@ struct fsmonitor_batch; struct fsmonitor_token_data; +/* + * Create a new batch of path(s). The returned batch is considered + * private and not linked into the fsmonitor daemon state. The caller + * should fill this batch with one or more paths and then publish it. + */ +struct fsmonitor_batch *fsmonitor_batch__new(void); + +/* + * Free the list of batches starting with this one. + */ +void fsmonitor_batch__free_list(struct fsmonitor_batch *batch); + +/* + * Add this path to this batch of modified files. + * + * The batch should be private and NOT (yet) linked into the fsmonitor + * daemon state and therefore not yet visible to worker threads and so + * no locking is required. + */ +void fsmonitor_batch__add_path(struct fsmonitor_batch *batch, const char *path); + struct fsmonitor_daemon_backend_data; /* opaque platform-specific data */ struct fsmonitor_daemon_state { @@ -91,5 +112,24 @@ enum fsmonitor_path_type fsmonitor_classify_path_absolute( struct fsmonitor_daemon_state *state, const char *path); +/* + * Prepend the this batch of path(s) onto the list of batches associated + * with the current token. This makes the batch visible to worker threads. + * + * The caller no longer owns the batch and must not free it. + * + * Wake up the client threads waiting on these cookies. + */ +void fsmonitor_publish(struct fsmonitor_daemon_state *state, + struct fsmonitor_batch *batch, + const struct string_list *cookie_names); + +/* + * If the platform-specific layer loses sync with the filesystem, + * it should call this to invalidate cached data and abort waiting + * threads. + */ +void fsmonitor_force_resync(struct fsmonitor_daemon_state *state); + #endif /* HAVE_FSMONITOR_DAEMON_BACKEND */ #endif /* FSMONITOR_DAEMON_H */ From dd378b45fa5b5a9883693e70269c7883fa8d0361 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 17 Dec 2020 13:34:30 -0500 Subject: [PATCH 070/110] compat/fsmonitor/fsm-listen-win32: implement FSMonitor backend on Windows Teach the win32 backend to register a watch on the working tree root directory (recursively). Also watch the if it is not inside the working tree. And to collect path change notifications into batches and publish. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-listen-win32.c | 565 ++++++++++++++++++++++++++++ 1 file changed, 565 insertions(+) diff --git a/compat/fsmonitor/fsm-listen-win32.c b/compat/fsmonitor/fsm-listen-win32.c index 916cbea254f8cf..c2d11acbc1ef7d 100644 --- a/compat/fsmonitor/fsm-listen-win32.c +++ b/compat/fsmonitor/fsm-listen-win32.c @@ -2,20 +2,585 @@ #include "config.h" #include "fsmonitor.h" #include "fsm-listen.h" +#include "fsmonitor--daemon.h" + +/* + * The documentation of ReadDirectoryChangesW() states that the maximum + * buffer size is 64K when the monitored directory is remote. + * + * Larger buffers may be used when the monitored directory is local and + * will help us receive events faster from the kernel and avoid dropped + * events. + * + * So we try to use a very large buffer and silently fallback to 64K if + * we get an error. + */ +#define MAX_RDCW_BUF_FALLBACK (65536) +#define MAX_RDCW_BUF (65536 * 8) + +struct one_watch +{ + char buffer[MAX_RDCW_BUF]; + DWORD buf_len; + DWORD count; + + struct strbuf path; + HANDLE hDir; + HANDLE hEvent; + OVERLAPPED overlapped; + + /* + * Is there an active ReadDirectoryChangesW() call pending. If so, we + * need to later call GetOverlappedResult() and possibly CancelIoEx(). + */ + BOOL is_active; +}; + +struct fsmonitor_daemon_backend_data +{ + struct one_watch *watch_worktree; + struct one_watch *watch_gitdir; + + HANDLE hEventShutdown; + + HANDLE hListener[3]; /* we don't own these handles */ +#define LISTENER_SHUTDOWN 0 +#define LISTENER_HAVE_DATA_WORKTREE 1 +#define LISTENER_HAVE_DATA_GITDIR 2 + int nr_listener_handles; +}; + +/* + * Convert the WCHAR path from the notification into UTF8 and + * then normalize it. + */ +static int normalize_path_in_utf8(FILE_NOTIFY_INFORMATION *info, + struct strbuf *normalized_path) +{ + int reserve; + int len = 0; + + strbuf_reset(normalized_path); + if (!info->FileNameLength) + goto normalize; + + /* + * Pre-reserve enough space in the UTF8 buffer for + * each Unicode WCHAR character to be mapped into a + * sequence of 2 UTF8 characters. That should let us + * avoid ERROR_INSUFFICIENT_BUFFER 99.9+% of the time. + */ + reserve = info->FileNameLength + 1; + strbuf_grow(normalized_path, reserve); + + for (;;) { + len = WideCharToMultiByte(CP_UTF8, 0, info->FileName, + info->FileNameLength / sizeof(WCHAR), + normalized_path->buf, + strbuf_avail(normalized_path) - 1, + NULL, NULL); + if (len > 0) + goto normalize; + if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) { + error("[GLE %ld] could not convert path to UTF-8: '%.*ls'", + GetLastError(), + (int)(info->FileNameLength / sizeof(WCHAR)), + info->FileName); + return -1; + } + + strbuf_grow(normalized_path, + strbuf_avail(normalized_path) + reserve); + } + +normalize: + strbuf_setlen(normalized_path, len); + return strbuf_normalize_path(normalized_path); +} void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) { + SetEvent(state->backend_data->hListener[LISTENER_SHUTDOWN]); +} + +static struct one_watch *create_watch(struct fsmonitor_daemon_state *state, + const char *path) +{ + struct one_watch *watch = NULL; + DWORD desired_access = FILE_LIST_DIRECTORY; + DWORD share_mode = + FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE; + HANDLE hDir; + wchar_t wpath[MAX_PATH]; + + if (xutftowcs_path(wpath, path) < 0) { + error(_("could not convert to wide characters: '%s'"), path); + return NULL; + } + + hDir = CreateFileW(wpath, + desired_access, share_mode, NULL, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED, + NULL); + if (hDir == INVALID_HANDLE_VALUE) { + error(_("[GLE %ld] could not watch '%s'"), + GetLastError(), path); + return NULL; + } + + CALLOC_ARRAY(watch, 1); + + watch->buf_len = sizeof(watch->buffer); /* assume full MAX_RDCW_BUF */ + + strbuf_init(&watch->path, 0); + strbuf_addstr(&watch->path, path); + + watch->hDir = hDir; + watch->hEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + + return watch; +} + +static void destroy_watch(struct one_watch *watch) +{ + if (!watch) + return; + + strbuf_release(&watch->path); + if (watch->hDir != INVALID_HANDLE_VALUE) + CloseHandle(watch->hDir); + if (watch->hEvent != INVALID_HANDLE_VALUE) + CloseHandle(watch->hEvent); + + free(watch); +} + +static int start_rdcw_watch(struct fsmonitor_daemon_backend_data *data, + struct one_watch *watch) +{ + DWORD dwNotifyFilter = + FILE_NOTIFY_CHANGE_FILE_NAME | + FILE_NOTIFY_CHANGE_DIR_NAME | + FILE_NOTIFY_CHANGE_ATTRIBUTES | + FILE_NOTIFY_CHANGE_SIZE | + FILE_NOTIFY_CHANGE_LAST_WRITE | + FILE_NOTIFY_CHANGE_CREATION; + + ResetEvent(watch->hEvent); + + memset(&watch->overlapped, 0, sizeof(watch->overlapped)); + watch->overlapped.hEvent = watch->hEvent; + + /* + * Queue an async call using Overlapped IO. This returns immediately. + * Our event handle will be signalled when the real result is available. + * + * The return value here just means that we successfully queued it. + * We won't know if the Read...() actually produces data until later. + */ + watch->is_active = ReadDirectoryChangesW( + watch->hDir, watch->buffer, watch->buf_len, TRUE, + dwNotifyFilter, &watch->count, &watch->overlapped, NULL); + + if (watch->is_active) + return 0; + + error("ReadDirectoryChangedW failed on '%s' [GLE %ld]", + watch->path.buf, GetLastError()); + return -1; +} + +static int recv_rdcw_watch(struct one_watch *watch) +{ + DWORD gle; + + watch->is_active = FALSE; + + /* + * The overlapped result is ready. If the Read...() was successful + * we finally receive the actual result into our buffer. + */ + if (GetOverlappedResult(watch->hDir, &watch->overlapped, &watch->count, + TRUE)) + return 0; + + gle = GetLastError(); + if (gle == ERROR_INVALID_PARAMETER && + /* + * The kernel throws an invalid parameter error when our + * buffer is too big and we are pointed at a remote + * directory (and possibly for other reasons). Quietly + * set it down and try again. + * + * See note about MAX_RDCW_BUF at the top. + */ + watch->buf_len > MAX_RDCW_BUF_FALLBACK) { + watch->buf_len = MAX_RDCW_BUF_FALLBACK; + return -2; + } + + /* + * NEEDSWORK: If an external is deleted, the above + * returns an error. I'm not sure that there's anything that + * we can do here other than failing -- the /.git + * link file would be broken anyway. We might try to check + * for that and return a better error message, but I'm not + * sure it is worth it. + */ + + error("GetOverlappedResult failed on '%s' [GLE %ld]", + watch->path.buf, gle); + return -1; +} + +static void cancel_rdcw_watch(struct one_watch *watch) +{ + DWORD count; + + if (!watch || !watch->is_active) + return; + + /* + * The calls to ReadDirectoryChangesW() and GetOverlappedResult() + * form a "pair" (my term) where we queue an IO and promise to + * hang around and wait for the kernel to give us the result. + * + * If for some reason after we queue the IO, we have to quit + * or otherwise not stick around for the second half, we must + * tell the kernel to abort the IO. This prevents the kernel + * from writing to our buffer and/or signalling our event + * after we free them. + * + * (Ask me how much fun it was to track that one down). + */ + CancelIoEx(watch->hDir, &watch->overlapped); + GetOverlappedResult(watch->hDir, &watch->overlapped, &count, TRUE); + watch->is_active = FALSE; +} + +/* + * Process filesystem events that happen anywhere (recursively) under the + * root directory. For a normal working directory, this includes + * both version controlled files and the contents of the .git/ directory. + * + * If /.git is a file, then we only see events for the file + * itself. + */ +static int process_worktree_events(struct fsmonitor_daemon_state *state) +{ + struct fsmonitor_daemon_backend_data *data = state->backend_data; + struct one_watch *watch = data->watch_worktree; + struct strbuf path = STRBUF_INIT; + struct string_list cookie_list = STRING_LIST_INIT_DUP; + struct fsmonitor_batch *batch = NULL; + const char *p = watch->buffer; + + /* + * If the kernel gets more events than will fit in the kernel + * buffer associated with our RDCW handle, it drops them and + * returns a count of zero. + * + * Yes, the call returns WITHOUT error and with length zero. + * This is the documented behavior. (My testing has confirmed + * that it also sets the last error to ERROR_NOTIFY_ENUM_DIR, + * but we do not rely on that since the function did not + * return an error and it is not documented.) + * + * (The "overflow" case is not ambiguous with the "no data" case + * because we did an INFINITE wait.) + * + * This means we have a gap in coverage. Tell the daemon layer + * to resync. + */ + if (!watch->count) { + trace2_data_string("fsmonitor", NULL, "fsm-listen/kernel", + "overflow"); + fsmonitor_force_resync(state); + return LISTENER_HAVE_DATA_WORKTREE; + } + + /* + * On Windows, `info` contains an "array" of paths that are + * relative to the root of whichever directory handle received + * the event. + */ + for (;;) { + FILE_NOTIFY_INFORMATION *info = (void *)p; + const char *slash; + enum fsmonitor_path_type t; + + strbuf_reset(&path); + if (normalize_path_in_utf8(info, &path) == -1) + goto skip_this_path; + + t = fsmonitor_classify_path_workdir_relative(path.buf); + + switch (t) { + case IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX: + /* special case cookie files within .git */ + + /* Use just the filename of the cookie file. */ + slash = find_last_dir_sep(path.buf); + string_list_append(&cookie_list, + slash ? slash + 1 : path.buf); + break; + + case IS_INSIDE_DOT_GIT: + /* ignore everything inside of "/.git/" */ + break; + + case IS_DOT_GIT: + /* "/.git" was deleted (or renamed away) */ + if ((info->Action == FILE_ACTION_REMOVED) || + (info->Action == FILE_ACTION_RENAMED_OLD_NAME)) { + trace2_data_string("fsmonitor", NULL, + "fsm-listen/dotgit", + "removed"); + goto force_shutdown; + } + break; + + case IS_WORKDIR_PATH: + /* queue normal pathname */ + if (!batch) + batch = fsmonitor_batch__new(); + fsmonitor_batch__add_path(batch, path.buf); + break; + + case IS_GITDIR: + case IS_INSIDE_GITDIR: + case IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX: + default: + BUG("unexpected path classification '%d' for '%s'", + t, path.buf); + } + +skip_this_path: + if (!info->NextEntryOffset) + break; + p += info->NextEntryOffset; + } + + fsmonitor_publish(state, batch, &cookie_list); + batch = NULL; + string_list_clear(&cookie_list, 0); + strbuf_release(&path); + return LISTENER_HAVE_DATA_WORKTREE; + +force_shutdown: + fsmonitor_batch__free_list(batch); + string_list_clear(&cookie_list, 0); + strbuf_release(&path); + return LISTENER_SHUTDOWN; +} + +/* + * Process filesystem events that happened anywhere (recursively) under the + * external (such as non-primary worktrees or submodules). + * We only care about cookie files that our client threads created here. + * + * Note that we DO NOT get filesystem events on the external + * itself (it is not inside something that we are watching). In particular, + * we do not get an event if the external is deleted. + */ +static int process_gitdir_events(struct fsmonitor_daemon_state *state) +{ + struct fsmonitor_daemon_backend_data *data = state->backend_data; + struct one_watch *watch = data->watch_gitdir; + struct strbuf path = STRBUF_INIT; + struct string_list cookie_list = STRING_LIST_INIT_DUP; + const char *p = watch->buffer; + + if (!watch->count) { + trace2_data_string("fsmonitor", NULL, "fsm-listen/kernel", + "overflow"); + fsmonitor_force_resync(state); + return LISTENER_HAVE_DATA_GITDIR; + } + + for (;;) { + FILE_NOTIFY_INFORMATION *info = (void *)p; + const char *slash; + enum fsmonitor_path_type t; + + strbuf_reset(&path); + if (normalize_path_in_utf8(info, &path) == -1) + goto skip_this_path; + + t = fsmonitor_classify_path_gitdir_relative(path.buf); + + switch (t) { + case IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX: + /* special case cookie files within gitdir */ + + /* Use just the filename of the cookie file. */ + slash = find_last_dir_sep(path.buf); + string_list_append(&cookie_list, + slash ? slash + 1 : path.buf); + break; + + case IS_INSIDE_GITDIR: + goto skip_this_path; + + default: + BUG("unexpected path classification '%d' for '%s'", + t, path.buf); + } + +skip_this_path: + if (!info->NextEntryOffset) + break; + p += info->NextEntryOffset; + } + + fsmonitor_publish(state, NULL, &cookie_list); + string_list_clear(&cookie_list, 0); + strbuf_release(&path); + return LISTENER_HAVE_DATA_GITDIR; } void fsm_listen__loop(struct fsmonitor_daemon_state *state) { + struct fsmonitor_daemon_backend_data *data = state->backend_data; + DWORD dwWait; + int result; + + state->error_code = 0; + + if (start_rdcw_watch(data, data->watch_worktree) == -1) + goto force_error_stop; + + if (data->watch_gitdir && + start_rdcw_watch(data, data->watch_gitdir) == -1) + goto force_error_stop; + + for (;;) { + dwWait = WaitForMultipleObjects(data->nr_listener_handles, + data->hListener, + FALSE, INFINITE); + + if (dwWait == WAIT_OBJECT_0 + LISTENER_HAVE_DATA_WORKTREE) { + result = recv_rdcw_watch(data->watch_worktree); + if (result == -1) { + /* hard error */ + goto force_error_stop; + } + if (result == -2) { + /* retryable error */ + if (start_rdcw_watch(data, data->watch_worktree) == -1) + goto force_error_stop; + continue; + } + + /* have data */ + if (process_worktree_events(state) == LISTENER_SHUTDOWN) + goto force_shutdown; + if (start_rdcw_watch(data, data->watch_worktree) == -1) + goto force_error_stop; + continue; + } + + if (dwWait == WAIT_OBJECT_0 + LISTENER_HAVE_DATA_GITDIR) { + result = recv_rdcw_watch(data->watch_gitdir); + if (result == -1) { + /* hard error */ + goto force_error_stop; + } + if (result == -2) { + /* retryable error */ + if (start_rdcw_watch(data, data->watch_gitdir) == -1) + goto force_error_stop; + continue; + } + + /* have data */ + if (process_gitdir_events(state) == LISTENER_SHUTDOWN) + goto force_shutdown; + if (start_rdcw_watch(data, data->watch_gitdir) == -1) + goto force_error_stop; + continue; + } + + if (dwWait == WAIT_OBJECT_0 + LISTENER_SHUTDOWN) + goto clean_shutdown; + + error(_("could not read directory changes [GLE %ld]"), + GetLastError()); + goto force_error_stop; + } + +force_error_stop: + state->error_code = -1; + +force_shutdown: + /* + * Tell the IPC thead pool to stop (which completes the await + * in the main thread (which will also signal this thread (if + * we are still alive))). + */ + ipc_server_stop_async(state->ipc_server_data); + +clean_shutdown: + cancel_rdcw_watch(data->watch_worktree); + cancel_rdcw_watch(data->watch_gitdir); } int fsm_listen__ctor(struct fsmonitor_daemon_state *state) { + struct fsmonitor_daemon_backend_data *data; + + CALLOC_ARRAY(data, 1); + + data->hEventShutdown = CreateEvent(NULL, TRUE, FALSE, NULL); + + data->watch_worktree = create_watch(state, + state->path_worktree_watch.buf); + if (!data->watch_worktree) + goto failed; + + if (state->nr_paths_watching > 1) { + data->watch_gitdir = create_watch(state, + state->path_gitdir_watch.buf); + if (!data->watch_gitdir) + goto failed; + } + + data->hListener[LISTENER_SHUTDOWN] = data->hEventShutdown; + data->nr_listener_handles++; + + data->hListener[LISTENER_HAVE_DATA_WORKTREE] = + data->watch_worktree->hEvent; + data->nr_listener_handles++; + + if (data->watch_gitdir) { + data->hListener[LISTENER_HAVE_DATA_GITDIR] = + data->watch_gitdir->hEvent; + data->nr_listener_handles++; + } + + state->backend_data = data; + return 0; + +failed: + CloseHandle(data->hEventShutdown); + destroy_watch(data->watch_worktree); + destroy_watch(data->watch_gitdir); + return -1; } void fsm_listen__dtor(struct fsmonitor_daemon_state *state) { + struct fsmonitor_daemon_backend_data *data; + + if (!state || !state->backend_data) + return; + + data = state->backend_data; + + CloseHandle(data->hEventShutdown); + destroy_watch(data->watch_worktree); + destroy_watch(data->watch_gitdir); + + FREE_AND_NULL(state->backend_data); } From 7aa75f6b6baa97d37571adef13ec2d3aeafc670d Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 18 Dec 2020 11:09:58 -0500 Subject: [PATCH 071/110] compat/fsmonitor/fsm-listen-darwin: add macos header files for FSEvent Include MacOS system declarations to allow us to use FSEvent and CoreFoundation APIs. We need GCC and clang versions because of compiler and header file conflicts. While it is quite possible to #include Apple's CoreServices.h when compiling C source code with clang, trying to build it with GCC currently fails with this error: In file included from /Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/System/Library/Frameworks/Security.framework/Headers/AuthSession.h:32, from /Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/System/Library/Frameworks/Security.framework/Headers/Security.h:42, from /Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/System/Library/Frameworks/CoreServices.framework/Frameworks/OSServices.framework/Headers/CSIdentity.h:43, from /Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/System/Library/Frameworks/CoreServices.framework/Frameworks/OSServices.framework/Headers/OSServices.h:29, from /Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Headers/IconsCore.h:23, from /Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Headers/LaunchServices.h:23, from /Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/System/Library/Frameworks/CoreServices.framework/Headers/CoreServices.h:45, /Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/System/Library/Frameworks/Security.framework/Headers/Authorization.h:193:7: error: variably modified 'bytes' at file scope 193 | char bytes[kAuthorizationExternalFormLength]; | ^~~~~ The underlying reason is that GCC (rightfully) objects that an `enum` value such as `kAuthorizationExternalFormLength` is not a constant (because it is not, the preprocessor has no knowledge of it, only the actual C compiler does) and can therefore not be used to define the size of a C array. This is a known problem and tracked in GCC's bug tracker: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93082 In the meantime, let's not block things and go the slightly ugly route of declaring/defining the FSEvents constants, data structures and functions that we need, so that we can avoid above-mentioned issue. Let's do this _only_ for GCC, though, so that the CI/PR builds (which build both with clang and with GCC) can guarantee that we _are_ using the correct data types. Signed-off-by: Johannes Schindelin Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-listen-darwin.c | 96 ++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/compat/fsmonitor/fsm-listen-darwin.c b/compat/fsmonitor/fsm-listen-darwin.c index c84e3344ab99b4..f424253d3eb037 100644 --- a/compat/fsmonitor/fsm-listen-darwin.c +++ b/compat/fsmonitor/fsm-listen-darwin.c @@ -1,3 +1,99 @@ +#if defined(__GNUC__) +/* + * It is possible to #include CoreFoundation/CoreFoundation.h when compiling + * with clang, but not with GCC as of time of writing. + * + * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93082 for details. + */ +typedef unsigned int FSEventStreamCreateFlags; +#define kFSEventStreamEventFlagNone 0x00000000 +#define kFSEventStreamEventFlagMustScanSubDirs 0x00000001 +#define kFSEventStreamEventFlagUserDropped 0x00000002 +#define kFSEventStreamEventFlagKernelDropped 0x00000004 +#define kFSEventStreamEventFlagEventIdsWrapped 0x00000008 +#define kFSEventStreamEventFlagHistoryDone 0x00000010 +#define kFSEventStreamEventFlagRootChanged 0x00000020 +#define kFSEventStreamEventFlagMount 0x00000040 +#define kFSEventStreamEventFlagUnmount 0x00000080 +#define kFSEventStreamEventFlagItemCreated 0x00000100 +#define kFSEventStreamEventFlagItemRemoved 0x00000200 +#define kFSEventStreamEventFlagItemInodeMetaMod 0x00000400 +#define kFSEventStreamEventFlagItemRenamed 0x00000800 +#define kFSEventStreamEventFlagItemModified 0x00001000 +#define kFSEventStreamEventFlagItemFinderInfoMod 0x00002000 +#define kFSEventStreamEventFlagItemChangeOwner 0x00004000 +#define kFSEventStreamEventFlagItemXattrMod 0x00008000 +#define kFSEventStreamEventFlagItemIsFile 0x00010000 +#define kFSEventStreamEventFlagItemIsDir 0x00020000 +#define kFSEventStreamEventFlagItemIsSymlink 0x00040000 +#define kFSEventStreamEventFlagOwnEvent 0x00080000 +#define kFSEventStreamEventFlagItemIsHardlink 0x00100000 +#define kFSEventStreamEventFlagItemIsLastHardlink 0x00200000 +#define kFSEventStreamEventFlagItemCloned 0x00400000 + +typedef struct __FSEventStream *FSEventStreamRef; +typedef const FSEventStreamRef ConstFSEventStreamRef; + +typedef unsigned int CFStringEncoding; +#define kCFStringEncodingUTF8 0x08000100 + +typedef const struct __CFString *CFStringRef; +typedef const struct __CFArray *CFArrayRef; +typedef const struct __CFRunLoop *CFRunLoopRef; + +struct FSEventStreamContext { + long long version; + void *cb_data, *retain, *release, *copy_description; +}; + +typedef struct FSEventStreamContext FSEventStreamContext; +typedef unsigned int FSEventStreamEventFlags; +#define kFSEventStreamCreateFlagNoDefer 0x02 +#define kFSEventStreamCreateFlagWatchRoot 0x04 +#define kFSEventStreamCreateFlagFileEvents 0x10 + +typedef unsigned long long FSEventStreamEventId; +#define kFSEventStreamEventIdSinceNow 0xFFFFFFFFFFFFFFFFULL + +typedef void (*FSEventStreamCallback)(ConstFSEventStreamRef streamRef, + void *context, + __SIZE_TYPE__ num_of_events, + void *event_paths, + const FSEventStreamEventFlags event_flags[], + const FSEventStreamEventId event_ids[]); +typedef double CFTimeInterval; +FSEventStreamRef FSEventStreamCreate(void *allocator, + FSEventStreamCallback callback, + FSEventStreamContext *context, + CFArrayRef paths_to_watch, + FSEventStreamEventId since_when, + CFTimeInterval latency, + FSEventStreamCreateFlags flags); +CFStringRef CFStringCreateWithCString(void *allocator, const char *string, + CFStringEncoding encoding); +CFArrayRef CFArrayCreate(void *allocator, const void **items, long long count, + void *callbacks); +void CFRunLoopRun(void); +void CFRunLoopStop(CFRunLoopRef run_loop); +CFRunLoopRef CFRunLoopGetCurrent(void); +extern CFStringRef kCFRunLoopDefaultMode; +void FSEventStreamScheduleWithRunLoop(FSEventStreamRef stream, + CFRunLoopRef run_loop, + CFStringRef run_loop_mode); +unsigned char FSEventStreamStart(FSEventStreamRef stream); +void FSEventStreamStop(FSEventStreamRef stream); +void FSEventStreamInvalidate(FSEventStreamRef stream); +void FSEventStreamRelease(FSEventStreamRef stream); +#else +/* + * Let Apple's headers declare `isalnum()` first, before + * Git's headers override it via a constant + */ +#include +#include +#include +#endif + #include "cache.h" #include "fsmonitor.h" #include "fsm-listen.h" From 38560a3b5a8fa5402e9be4ba84fb3b17144138cb Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 18 Dec 2020 11:22:25 -0500 Subject: [PATCH 072/110] compat/fsmonitor/fsm-listen-darwin: implement FSEvent listener on MacOS Implement file system event listener on MacOS using FSEvent, CoreFoundation, and CoreServices. Co-authored-by: Kevin Willford Co-authored-by: Johannes Schindelin Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-listen-darwin.c | 382 ++++++++++++++++++++++++++- 1 file changed, 381 insertions(+), 1 deletion(-) diff --git a/compat/fsmonitor/fsm-listen-darwin.c b/compat/fsmonitor/fsm-listen-darwin.c index f424253d3eb037..2aefdc14d8917b 100644 --- a/compat/fsmonitor/fsm-listen-darwin.c +++ b/compat/fsmonitor/fsm-listen-darwin.c @@ -1,4 +1,4 @@ -#if defined(__GNUC__) +#ifndef __clang__ /* * It is possible to #include CoreFoundation/CoreFoundation.h when compiling * with clang, but not with GCC as of time of writing. @@ -97,20 +97,400 @@ void FSEventStreamRelease(FSEventStreamRef stream); #include "cache.h" #include "fsmonitor.h" #include "fsm-listen.h" +#include "fsmonitor--daemon.h" + +struct fsmonitor_daemon_backend_data +{ + CFStringRef cfsr_worktree_path; + CFStringRef cfsr_gitdir_path; + + CFArrayRef cfar_paths_to_watch; + int nr_paths_watching; + + FSEventStreamRef stream; + + CFRunLoopRef rl; + + enum shutdown_style { + SHUTDOWN_EVENT = 0, + FORCE_SHUTDOWN, + FORCE_ERROR_STOP, + } shutdown_style; + + unsigned int stream_scheduled:1; + unsigned int stream_started:1; +}; + +static void log_flags_set(const char *path, const FSEventStreamEventFlags flag) +{ + struct strbuf msg = STRBUF_INIT; + + if (flag & kFSEventStreamEventFlagMustScanSubDirs) + strbuf_addstr(&msg, "MustScanSubDirs|"); + if (flag & kFSEventStreamEventFlagUserDropped) + strbuf_addstr(&msg, "UserDropped|"); + if (flag & kFSEventStreamEventFlagKernelDropped) + strbuf_addstr(&msg, "KernelDropped|"); + if (flag & kFSEventStreamEventFlagEventIdsWrapped) + strbuf_addstr(&msg, "EventIdsWrapped|"); + if (flag & kFSEventStreamEventFlagHistoryDone) + strbuf_addstr(&msg, "HistoryDone|"); + if (flag & kFSEventStreamEventFlagRootChanged) + strbuf_addstr(&msg, "RootChanged|"); + if (flag & kFSEventStreamEventFlagMount) + strbuf_addstr(&msg, "Mount|"); + if (flag & kFSEventStreamEventFlagUnmount) + strbuf_addstr(&msg, "Unmount|"); + if (flag & kFSEventStreamEventFlagItemChangeOwner) + strbuf_addstr(&msg, "ItemChangeOwner|"); + if (flag & kFSEventStreamEventFlagItemCreated) + strbuf_addstr(&msg, "ItemCreated|"); + if (flag & kFSEventStreamEventFlagItemFinderInfoMod) + strbuf_addstr(&msg, "ItemFinderInfoMod|"); + if (flag & kFSEventStreamEventFlagItemInodeMetaMod) + strbuf_addstr(&msg, "ItemInodeMetaMod|"); + if (flag & kFSEventStreamEventFlagItemIsDir) + strbuf_addstr(&msg, "ItemIsDir|"); + if (flag & kFSEventStreamEventFlagItemIsFile) + strbuf_addstr(&msg, "ItemIsFile|"); + if (flag & kFSEventStreamEventFlagItemIsHardlink) + strbuf_addstr(&msg, "ItemIsHardlink|"); + if (flag & kFSEventStreamEventFlagItemIsLastHardlink) + strbuf_addstr(&msg, "ItemIsLastHardlink|"); + if (flag & kFSEventStreamEventFlagItemIsSymlink) + strbuf_addstr(&msg, "ItemIsSymlink|"); + if (flag & kFSEventStreamEventFlagItemModified) + strbuf_addstr(&msg, "ItemModified|"); + if (flag & kFSEventStreamEventFlagItemRemoved) + strbuf_addstr(&msg, "ItemRemoved|"); + if (flag & kFSEventStreamEventFlagItemRenamed) + strbuf_addstr(&msg, "ItemRenamed|"); + if (flag & kFSEventStreamEventFlagItemXattrMod) + strbuf_addstr(&msg, "ItemXattrMod|"); + if (flag & kFSEventStreamEventFlagOwnEvent) + strbuf_addstr(&msg, "OwnEvent|"); + if (flag & kFSEventStreamEventFlagItemCloned) + strbuf_addstr(&msg, "ItemCloned|"); + + trace_printf_key(&trace_fsmonitor, "fsevent: '%s', flags=%u %s", + path, flag, msg.buf); + + strbuf_release(&msg); +} + +static int ef_is_root_delete(const FSEventStreamEventFlags ef) +{ + return (ef & kFSEventStreamEventFlagItemIsDir && + ef & kFSEventStreamEventFlagItemRemoved); +} + +static int ef_is_root_renamed(const FSEventStreamEventFlags ef) +{ + return (ef & kFSEventStreamEventFlagItemIsDir && + ef & kFSEventStreamEventFlagItemRenamed); +} + +static int ef_is_dropped(const FSEventStreamEventFlags ef) +{ + return (ef & kFSEventStreamEventFlagMustScanSubDirs || + ef & kFSEventStreamEventFlagKernelDropped || + ef & kFSEventStreamEventFlagUserDropped); +} + +static void fsevent_callback(ConstFSEventStreamRef streamRef, + void *ctx, + size_t num_of_events, + void *event_paths, + const FSEventStreamEventFlags event_flags[], + const FSEventStreamEventId event_ids[]) +{ + struct fsmonitor_daemon_state *state = ctx; + struct fsmonitor_daemon_backend_data *data = state->backend_data; + char **paths = (char **)event_paths; + struct fsmonitor_batch *batch = NULL; + struct string_list cookie_list = STRING_LIST_INIT_DUP; + const char *path_k; + const char *slash; + int k; + struct strbuf tmp = STRBUF_INIT; + + /* + * Build a list of all filesystem changes into a private/local + * list and without holding any locks. + */ + for (k = 0; k < num_of_events; k++) { + /* + * On Mac, we receive an array of absolute paths. + */ + path_k = paths[k]; + + /* + * If you want to debug FSEvents, log them to GIT_TRACE_FSMONITOR. + * Please don't log them to Trace2. + * + * trace_printf_key(&trace_fsmonitor, "Path: '%s'", path_k); + */ + + /* + * If event[k] is marked as dropped, we assume that we have + * lost sync with the filesystem and should flush our cached + * data. We need to: + * + * [1] Abort/wake any client threads waiting for a cookie and + * flush the cached state data (the current token), and + * create a new token. + * + * [2] Discard the batch that we were locally building (since + * they are conceptually relative to the just flushed + * token). + */ + if (ef_is_dropped(event_flags[k])) { + if (trace_pass_fl(&trace_fsmonitor)) + log_flags_set(path_k, event_flags[k]); + + fsmonitor_force_resync(state); + fsmonitor_batch__free_list(batch); + string_list_clear(&cookie_list, 0); + + /* + * We assume that any events that we received + * in this callback after this dropped event + * may still be valid, so we continue rather + * than break. (And just in case there is a + * delete of ".git" hiding in there.) + */ + continue; + } + + switch (fsmonitor_classify_path_absolute(state, path_k)) { + + case IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX: + case IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX: + /* special case cookie files within .git or gitdir */ + + /* Use just the filename of the cookie file. */ + slash = find_last_dir_sep(path_k); + string_list_append(&cookie_list, + slash ? slash + 1 : path_k); + break; + + case IS_INSIDE_DOT_GIT: + case IS_INSIDE_GITDIR: + /* ignore all other paths inside of .git or gitdir */ + break; + + case IS_DOT_GIT: + case IS_GITDIR: + /* + * If .git directory is deleted or renamed away, + * we have to quit. + */ + if (ef_is_root_delete(event_flags[k])) { + trace_printf_key(&trace_fsmonitor, + "event: gitdir removed"); + goto force_shutdown; + } + if (ef_is_root_renamed(event_flags[k])) { + trace_printf_key(&trace_fsmonitor, + "event: gitdir renamed"); + goto force_shutdown; + } + break; + + case IS_WORKDIR_PATH: + /* try to queue normal pathnames */ + + if (trace_pass_fl(&trace_fsmonitor)) + log_flags_set(path_k, event_flags[k]); + + /* + * Because of the implicit "binning" (the + * kernel calls us at a given frequency) and + * de-duping (the kernel is free to combine + * multiple events for a given pathname), an + * individual fsevent could be marked as both + * a file and directory. Add it to the queue + * with both spellings so that the client will + * know how much to invalidate/refresh. + */ + + if (event_flags[k] & kFSEventStreamEventFlagItemIsFile) { + const char *rel = path_k + + state->path_worktree_watch.len + 1; + + if (!batch) + batch = fsmonitor_batch__new(); + fsmonitor_batch__add_path(batch, rel); + } + + if (event_flags[k] & kFSEventStreamEventFlagItemIsDir) { + const char *rel = path_k + + state->path_worktree_watch.len + 1; + + strbuf_reset(&tmp); + strbuf_addstr(&tmp, rel); + strbuf_addch(&tmp, '/'); + + if (!batch) + batch = fsmonitor_batch__new(); + fsmonitor_batch__add_path(batch, tmp.buf); + } + + break; + + case IS_OUTSIDE_CONE: + default: + trace_printf_key(&trace_fsmonitor, + "ignoring '%s'", path_k); + break; + } + } + + fsmonitor_publish(state, batch, &cookie_list); + string_list_clear(&cookie_list, 0); + strbuf_release(&tmp); + return; + +force_shutdown: + fsmonitor_batch__free_list(batch); + string_list_clear(&cookie_list, 0); + + data->shutdown_style = FORCE_SHUTDOWN; + CFRunLoopStop(data->rl); + strbuf_release(&tmp); + return; +} + +/* + * NEEDSWORK: Investigate the proper value for the `latency` argument + * in the call to `FSEventStreamCreate()`. I'm not sure that this + * needs to be a config setting or just something that we tune after + * some testing. + * + * With a latency of 0.1, I was seeing lots of dropped events during + * the "touch 100000" files test within t/perf/p7519, but with a + * latency of 0.001 I did not see any dropped events. So the + * "correct" value may be somewhere in between. + * + * https://developer.apple.com/documentation/coreservices/1443980-fseventstreamcreate + */ int fsm_listen__ctor(struct fsmonitor_daemon_state *state) { + FSEventStreamCreateFlags flags = kFSEventStreamCreateFlagNoDefer | + kFSEventStreamCreateFlagWatchRoot | + kFSEventStreamCreateFlagFileEvents; + FSEventStreamContext ctx = { + 0, + state, + NULL, + NULL, + NULL + }; + struct fsmonitor_daemon_backend_data *data; + const void *dir_array[2]; + + CALLOC_ARRAY(data, 1); + state->backend_data = data; + + data->cfsr_worktree_path = CFStringCreateWithCString( + NULL, state->path_worktree_watch.buf, kCFStringEncodingUTF8); + dir_array[data->nr_paths_watching++] = data->cfsr_worktree_path; + + if (state->nr_paths_watching > 1) { + data->cfsr_gitdir_path = CFStringCreateWithCString( + NULL, state->path_gitdir_watch.buf, + kCFStringEncodingUTF8); + dir_array[data->nr_paths_watching++] = data->cfsr_gitdir_path; + } + + data->cfar_paths_to_watch = CFArrayCreate(NULL, dir_array, + data->nr_paths_watching, + NULL); + data->stream = FSEventStreamCreate(NULL, fsevent_callback, &ctx, + data->cfar_paths_to_watch, + kFSEventStreamEventIdSinceNow, + 0.001, flags); + if (data->stream == NULL) + goto failed; + + /* + * `data->rl` needs to be set inside the listener thread. + */ + + return 0; + +failed: + error("Unable to create FSEventStream."); + + FREE_AND_NULL(state->backend_data); return -1; } void fsm_listen__dtor(struct fsmonitor_daemon_state *state) { + struct fsmonitor_daemon_backend_data *data; + + if (!state || !state->backend_data) + return; + + data = state->backend_data; + + if (data->stream) { + if (data->stream_started) + FSEventStreamStop(data->stream); + if (data->stream_scheduled) + FSEventStreamInvalidate(data->stream); + FSEventStreamRelease(data->stream); + } + + FREE_AND_NULL(state->backend_data); } void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) { + struct fsmonitor_daemon_backend_data *data; + + data = state->backend_data; + data->shutdown_style = SHUTDOWN_EVENT; + + CFRunLoopStop(data->rl); } void fsm_listen__loop(struct fsmonitor_daemon_state *state) { + struct fsmonitor_daemon_backend_data *data; + + data = state->backend_data; + + data->rl = CFRunLoopGetCurrent(); + + FSEventStreamScheduleWithRunLoop(data->stream, data->rl, kCFRunLoopDefaultMode); + data->stream_scheduled = 1; + + if (!FSEventStreamStart(data->stream)) { + error("Failed to start the FSEventStream"); + goto force_error_stop_without_loop; + } + data->stream_started = 1; + + CFRunLoopRun(); + + switch (data->shutdown_style) { + case FORCE_ERROR_STOP: + state->error_code = -1; + /* fall thru */ + case FORCE_SHUTDOWN: + ipc_server_stop_async(state->ipc_server_data); + /* fall thru */ + case SHUTDOWN_EVENT: + default: + break; + } + return; + +force_error_stop_without_loop: + state->error_code = -1; + ipc_server_stop_async(state->ipc_server_data); + return; } From cace837b534982136a1fefd370f1a04c510a6021 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 18 Dec 2020 11:43:16 -0500 Subject: [PATCH 073/110] fsmonitor--daemon: implement handle_client callback Teach fsmonitor--daemon to respond to IPC requests from client Git processes and respond with a list of modified pathnames relative to the provided token. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 312 +++++++++++++++++++++++++++++++++++- 1 file changed, 310 insertions(+), 2 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 34603f23053f61..4c88171e06c76c 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -7,6 +7,7 @@ #include "fsmonitor--daemon.h" #include "simple-ipc.h" #include "khash.h" +#include "pkt-line.h" static const char * const builtin_fsmonitor__daemon_usage[] = { N_("git fsmonitor--daemon start []"), @@ -351,6 +352,311 @@ void fsmonitor_force_resync(struct fsmonitor_daemon_state *state) pthread_mutex_unlock(&state->main_lock); } +/* + * Format an opaque token string to send to the client. + */ +static void with_lock__format_response_token( + struct strbuf *response_token, + const struct strbuf *response_token_id, + const struct fsmonitor_batch *batch) +{ + /* assert current thread holding state->main_lock */ + + strbuf_reset(response_token); + strbuf_addf(response_token, "builtin:%s:%"PRIu64, + response_token_id->buf, batch->batch_seq_nr); +} + +/* + * Parse an opaque token from the client. + * Returns -1 on error. + */ +static int fsmonitor_parse_client_token(const char *buf_token, + struct strbuf *requested_token_id, + uint64_t *seq_nr) +{ + const char *p; + char *p_end; + + strbuf_reset(requested_token_id); + *seq_nr = 0; + + if (!skip_prefix(buf_token, "builtin:", &p)) + return -1; + + while (*p && *p != ':') + strbuf_addch(requested_token_id, *p++); + if (!*p++) + return -1; + + *seq_nr = (uint64_t)strtoumax(p, &p_end, 10); + if (*p_end) + return -1; + + return 0; +} + +KHASH_INIT(str, const char *, int, 0, kh_str_hash_func, kh_str_hash_equal) + +static int do_handle_client(struct fsmonitor_daemon_state *state, + const char *command, + ipc_server_reply_cb *reply, + struct ipc_server_reply_data *reply_data) +{ + struct fsmonitor_token_data *token_data = NULL; + struct strbuf response_token = STRBUF_INIT; + struct strbuf requested_token_id = STRBUF_INIT; + struct strbuf payload = STRBUF_INIT; + uint64_t requested_oldest_seq_nr = 0; + uint64_t total_response_len = 0; + const char *p; + const struct fsmonitor_batch *batch_head; + const struct fsmonitor_batch *batch; + intmax_t count = 0, duplicates = 0; + kh_str_t *shown; + int hash_ret; + int do_trivial = 0; + int do_flush = 0; + + /* + * We expect `command` to be of the form: + * + * := quit NUL + * | flush NUL + * | NUL + * | NUL + */ + + if (!strcmp(command, "quit")) { + /* + * A client has requested over the socket/pipe that the + * daemon shutdown. + * + * Tell the IPC thread pool to shutdown (which completes + * the await in the main thread (which can stop the + * fsmonitor listener thread)). + * + * There is no reply to the client. + */ + return SIMPLE_IPC_QUIT; + + } else if (!strcmp(command, "flush")) { + /* + * Flush all of our cached data and generate a new token + * just like if we lost sync with the filesystem. + * + * Then send a trivial response using the new token. + */ + do_flush = 1; + do_trivial = 1; + + } else if (!skip_prefix(command, "builtin:", &p)) { + /* assume V1 timestamp or garbage */ + + char *p_end; + + strtoumax(command, &p_end, 10); + trace_printf_key(&trace_fsmonitor, + ((*p_end) ? + "fsmonitor: invalid command line '%s'" : + "fsmonitor: unsupported V1 protocol '%s'"), + command); + do_trivial = 1; + + } else { + /* We have "builtin:*" */ + if (fsmonitor_parse_client_token(command, &requested_token_id, + &requested_oldest_seq_nr)) { + trace_printf_key(&trace_fsmonitor, + "fsmonitor: invalid V2 protocol token '%s'", + command); + do_trivial = 1; + + } else { + /* + * We have a V2 valid token: + * "builtin::" + */ + } + } + + pthread_mutex_lock(&state->main_lock); + + if (!state->current_token_data) + BUG("fsmonitor state does not have a current token"); + + if (do_flush) + with_lock__do_force_resync(state); + + /* + * We mark the current head of the batch list as "pinned" so + * that the listener thread will treat this item as read-only + * (and prevent any more paths from being added to it) from + * now on. + */ + token_data = state->current_token_data; + batch_head = token_data->batch_head; + ((struct fsmonitor_batch *)batch_head)->pinned_time = time(NULL); + + /* + * FSMonitor Protocol V2 requires that we send a response header + * with a "new current token" and then all of the paths that changed + * since the "requested token". We send the seq_nr of the just-pinned + * head batch so that future requests from a client will be relative + * to it. + */ + with_lock__format_response_token(&response_token, + &token_data->token_id, batch_head); + + reply(reply_data, response_token.buf, response_token.len + 1); + total_response_len += response_token.len + 1; + + trace2_data_string("fsmonitor", the_repository, "response/token", + response_token.buf); + trace_printf_key(&trace_fsmonitor, "response token: %s", + response_token.buf); + + if (!do_trivial) { + if (strcmp(requested_token_id.buf, token_data->token_id.buf)) { + /* + * The client last spoke to a different daemon + * instance -OR- the daemon had to resync with + * the filesystem (and lost events), so reject. + */ + trace2_data_string("fsmonitor", the_repository, + "response/token", "different"); + do_trivial = 1; + + } else if (requested_oldest_seq_nr < + token_data->batch_tail->batch_seq_nr) { + /* + * The client wants older events than we have for + * this token_id. This means that the end of our + * batch list was truncated and we cannot give the + * client a complete snapshot relative to their + * request. + */ + trace_printf_key(&trace_fsmonitor, + "client requested truncated data"); + do_trivial = 1; + } + } + + if (do_trivial) { + pthread_mutex_unlock(&state->main_lock); + + reply(reply_data, "/", 2); + + trace2_data_intmax("fsmonitor", the_repository, + "response/trivial", 1); + + strbuf_release(&response_token); + strbuf_release(&requested_token_id); + return 0; + } + + /* + * We're going to hold onto a pointer to the current + * token-data while we walk the list of batches of files. + * During this time, we will NOT be under the lock. + * So we ref-count it. + * + * This allows the listener thread to continue prepending + * new batches of items to the token-data (which we'll ignore). + * + * AND it allows the listener thread to do a token-reset + * (and install a new `current_token_data`). + */ + token_data->client_ref_count++; + + pthread_mutex_unlock(&state->main_lock); + + /* + * The client request is relative to the token that they sent, + * so walk the batch list backwards from the current head back + * to the batch (sequence number) they named. + * + * We use khash to de-dup the list of pathnames. + * + * NEEDSWORK: each batch contains a list of interned strings, + * so we only need to do pointer comparisons here to build the + * hash table. Currently, we're still comparing the string + * values. + */ + shown = kh_init_str(); + for (batch = batch_head; + batch && batch->batch_seq_nr > requested_oldest_seq_nr; + batch = batch->next) { + size_t k; + + for (k = 0; k < batch->nr; k++) { + const char *s = batch->interned_paths[k]; + size_t s_len; + + if (kh_get_str(shown, s) != kh_end(shown)) + duplicates++; + else { + kh_put_str(shown, s, &hash_ret); + + trace_printf_key(&trace_fsmonitor, + "send[%"PRIuMAX"]: %s", + count, s); + + /* Each path gets written with a trailing NUL */ + s_len = strlen(s) + 1; + + if (payload.len + s_len >= + LARGE_PACKET_DATA_MAX) { + reply(reply_data, payload.buf, + payload.len); + total_response_len += payload.len; + strbuf_reset(&payload); + } + + strbuf_add(&payload, s, s_len); + count++; + } + } + } + + if (payload.len) { + reply(reply_data, payload.buf, payload.len); + total_response_len += payload.len; + } + + kh_release_str(shown); + + pthread_mutex_lock(&state->main_lock); + + if (token_data->client_ref_count > 0) + token_data->client_ref_count--; + + if (token_data->client_ref_count == 0) { + if (token_data != state->current_token_data) { + /* + * The listener thread did a token-reset while we were + * walking the batch list. Therefore, this token is + * stale and can be discarded completely. If we are + * the last reader thread using this token, we own + * that work. + */ + fsmonitor_free_token_data(token_data); + } + } + + pthread_mutex_unlock(&state->main_lock); + + trace2_data_intmax("fsmonitor", the_repository, "response/length", total_response_len); + trace2_data_intmax("fsmonitor", the_repository, "response/count/files", count); + trace2_data_intmax("fsmonitor", the_repository, "response/count/duplicates", duplicates); + + strbuf_release(&response_token); + strbuf_release(&requested_token_id); + strbuf_release(&payload); + + return 0; +} + static ipc_server_application_cb handle_client; static int handle_client(void *data, @@ -358,7 +664,7 @@ static int handle_client(void *data, ipc_server_reply_cb *reply, struct ipc_server_reply_data *reply_data) { - /* struct fsmonitor_daemon_state *state = data; */ + struct fsmonitor_daemon_state *state = data; int result; /* @@ -369,10 +675,12 @@ static int handle_client(void *data, if (command_len != strlen(command)) BUG("FSMonitor assumes text messages"); + trace_printf_key(&trace_fsmonitor, "requested token: %s", command); + trace2_region_enter("fsmonitor", "handle_client", the_repository); trace2_data_string("fsmonitor", the_repository, "request", command); - result = 0; /* TODO Do something here. */ + result = do_handle_client(state, command, reply, reply_data); trace2_region_leave("fsmonitor", "handle_client", the_repository); From 336e91833fd33f4a4f98c844305237bce1a15477 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 12 May 2021 08:40:50 -0400 Subject: [PATCH 074/110] help: include fsmonitor--daemon feature flag in version info Add the "feature: fsmonitor--daemon" message to the output of `git version --build-options`. The builtin FSMonitor is only available on certain platforms and even then only when certain Makefile flags are enabled, so print a message in the verbose version output when it is available. This can be used by test scripts for prereq testing. Granted, tests could just try `git fsmonitor--daemon status` and look for a 128 exit code or grep for a "not supported" message on stderr, but this is rather obscure. The main advantage is that the feature message will automatically appear in bug reports and other support requests. Signed-off-by: Jeff Hostetler --- help.c | 4 ++++ t/test-lib.sh | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/help.c b/help.c index 3c3bdec21356d9..e22ba1d246a5b0 100644 --- a/help.c +++ b/help.c @@ -11,6 +11,7 @@ #include "version.h" #include "refs.h" #include "parse-options.h" +#include "fsmonitor-ipc.h" struct category_description { uint32_t category; @@ -664,6 +665,9 @@ void get_version_info(struct strbuf *buf, int show_build_options) strbuf_addf(buf, "sizeof-size_t: %d\n", (int)sizeof(size_t)); strbuf_addf(buf, "shell-path: %s\n", SHELL_PATH); /* NEEDSWORK: also save and output GIT-BUILD_OPTIONS? */ + + if (fsmonitor_ipc__is_supported()) + strbuf_addstr(buf, "feature: fsmonitor--daemon\n"); } } diff --git a/t/test-lib.sh b/t/test-lib.sh index fd927bd9b6875e..964c3634d254ad 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -1745,3 +1745,9 @@ test_lazy_prereq REBASE_P ' # Tests that verify the scheduler integration must set this locally # to avoid errors. GIT_TEST_MAINT_SCHEDULER="none:exit 1" + +# Does this platform support `git fsmonitor--daemon` +# +test_lazy_prereq FSMONITOR_DAEMON ' + git version --build-options | grep "feature:" | grep "fsmonitor--daemon" +' From 370af40bc064775a24a51ebb42e445d9c93f59c4 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 11 May 2021 18:32:07 -0400 Subject: [PATCH 075/110] t/helper/fsmonitor-client: create IPC client to talk to FSMonitor Daemon Create an IPC client to send query and flush commands to the daemon. Signed-off-by: Jeff Hostetler --- Makefile | 1 + t/helper/test-fsmonitor-client.c | 121 +++++++++++++++++++++++++++++++ t/helper/test-tool.c | 1 + t/helper/test-tool.h | 1 + 4 files changed, 124 insertions(+) create mode 100644 t/helper/test-fsmonitor-client.c diff --git a/Makefile b/Makefile index bd7c807c4f7a3f..dca63dd2d55cc0 100644 --- a/Makefile +++ b/Makefile @@ -719,6 +719,7 @@ TEST_BUILTINS_OBJS += test-dump-split-index.o TEST_BUILTINS_OBJS += test-dump-untracked-cache.o TEST_BUILTINS_OBJS += test-example-decorate.o TEST_BUILTINS_OBJS += test-fast-rebase.o +TEST_BUILTINS_OBJS += test-fsmonitor-client.o TEST_BUILTINS_OBJS += test-genrandom.o TEST_BUILTINS_OBJS += test-genzeros.o TEST_BUILTINS_OBJS += test-getcwd.o diff --git a/t/helper/test-fsmonitor-client.c b/t/helper/test-fsmonitor-client.c new file mode 100644 index 00000000000000..f7a5b3a32fab86 --- /dev/null +++ b/t/helper/test-fsmonitor-client.c @@ -0,0 +1,121 @@ +/* + * test-fsmonitor-client.c: client code to send commands/requests to + * a `git fsmonitor--daemon` daemon. + */ + +#include "test-tool.h" +#include "cache.h" +#include "parse-options.h" +#include "fsmonitor-ipc.h" + +#ifndef HAVE_FSMONITOR_DAEMON_BACKEND +int cmd__fsmonitor_client(int argc, const char **argv) +{ + die("fsmonitor--daemon not available on this platform"); +} +#else + +/* + * Read the `.git/index` to get the last token written to the + * FSMonitor Index Extension. + */ +static const char *get_token_from_index(void) +{ + struct index_state *istate = the_repository->index; + + if (do_read_index(istate, the_repository->index_file, 0) < 0) + die("unable to read index file"); + if (!istate->fsmonitor_last_update) + die("index file does not have fsmonitor extension"); + + return istate->fsmonitor_last_update; +} + +/* + * Send an IPC query to a `git-fsmonitor--daemon` daemon and + * ask for the changes since the given token or from the last + * token in the index extension. + * + * This will implicitly start a daemon process if necessary. The + * daemon process will persist after we exit. + */ +static int do_send_query(const char *token) +{ + struct strbuf answer = STRBUF_INIT; + int ret; + + if (!token || !*token) + token = get_token_from_index(); + + ret = fsmonitor_ipc__send_query(token, &answer); + if (ret < 0) + die(_("could not query fsmonitor--daemon")); + + write_in_full(1, answer.buf, answer.len); + strbuf_release(&answer); + + return 0; +} + +/* + * Send a "flush" command to the `git-fsmonitor--daemon` (if running) + * and tell it to flush its cache. + * + * This feature is primarily used by the test suite to simulate a loss of + * sync with the filesystem where we miss kernel events. + */ +static int do_send_flush(void) +{ + struct strbuf answer = STRBUF_INIT; + int ret; + + ret = fsmonitor_ipc__send_command("flush", &answer); + if (ret) + return ret; + + write_in_full(1, answer.buf, answer.len); + strbuf_release(&answer); + + return 0; +} + +int cmd__fsmonitor_client(int argc, const char **argv) +{ + const char *subcmd; + const char *token = NULL; + + const char * const fsmonitor_client_usage[] = { + N_("test-helper fsmonitor-client query []"), + N_("test-helper fsmonitor-client flush"), + NULL, + }; + + struct option options[] = { + OPT_STRING(0, "token", &token, N_("token"), + N_("command token to send to the server")), + OPT_END() + }; + + if (argc < 2) + usage_with_options(fsmonitor_client_usage, options); + + if (argc == 2 && !strcmp(argv[1], "-h")) + usage_with_options(fsmonitor_client_usage, options); + + subcmd = argv[1]; + argv--; + argc++; + + argc = parse_options(argc, argv, NULL, options, fsmonitor_client_usage, 0); + + setup_git_directory(); + + if (!strcmp(subcmd, "query")) + return !!do_send_query(token); + + if (!strcmp(subcmd, "flush")) + return !!do_send_flush(); + + die("Unhandled subcommand: '%s'", subcmd); +} +#endif diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index 5efa225cc32949..7669b19177c0bc 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -32,6 +32,7 @@ static struct test_cmd cmds[] = { { "dump-untracked-cache", cmd__dump_untracked_cache }, { "example-decorate", cmd__example_decorate }, { "fast-rebase", cmd__fast_rebase }, + { "fsmonitor-client", cmd__fsmonitor_client }, { "genrandom", cmd__genrandom }, { "genzeros", cmd__genzeros }, { "getcwd", cmd__getcwd }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index b8712a5223958d..52b008e020006f 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -22,6 +22,7 @@ int cmd__dump_split_index(int argc, const char **argv); int cmd__dump_untracked_cache(int argc, const char **argv); int cmd__example_decorate(int argc, const char **argv); int cmd__fast_rebase(int argc, const char **argv); +int cmd__fsmonitor_client(int argc, const char **argv); int cmd__genrandom(int argc, const char **argv); int cmd__genzeros(int argc, const char **argv); int cmd__getcwd(int argc, const char **argv); From 6afcddb08b872d7456b38cd9f4d65bb8a71abb61 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 18 Dec 2020 12:31:37 -0500 Subject: [PATCH 076/110] t7527: create test for fsmonitor--daemon Signed-off-by: Jeff Hostetler --- t/t7527-builtin-fsmonitor.sh | 511 +++++++++++++++++++++++++++++++++++ 1 file changed, 511 insertions(+) create mode 100755 t/t7527-builtin-fsmonitor.sh diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh new file mode 100755 index 00000000000000..eea9ca1a309ed3 --- /dev/null +++ b/t/t7527-builtin-fsmonitor.sh @@ -0,0 +1,511 @@ +#!/bin/sh + +test_description='built-in file system watcher' + +. ./test-lib.sh + +if ! test_have_prereq FSMONITOR_DAEMON +then + skip_all="fsmonitor--daemon is not supported on this platform" + test_done +fi + +stop_daemon_delete_repo () { + r=$1 + git -C $r fsmonitor--daemon stop >/dev/null 2>/dev/null + rm -rf $1 + return 0 +} + +start_daemon () { + case "$#" in + 1) r="-C $1";; + *) r=""; + esac + + git $r fsmonitor--daemon start || return $? + git $r fsmonitor--daemon status || return $? + + return 0 +} + +# Is a Trace2 data event present with the given catetory and key? +# We do not care what the value is. +# +have_t2_data_event () { + c=$1 + k=$2 + + grep -e '"event":"data".*"category":"'"$c"'".*"key":"'"$k"'"' +} + +test_expect_success 'explicit daemon start and stop' ' + test_when_finished "stop_daemon_delete_repo test_explicit" && + + git init test_explicit && + start_daemon test_explicit && + + git -C test_explicit fsmonitor--daemon stop && + test_must_fail git -C test_explicit fsmonitor--daemon status +' + +test_expect_success 'implicit daemon start' ' + test_when_finished "stop_daemon_delete_repo test_implicit" && + + git init test_implicit && + test_must_fail git -C test_implicit fsmonitor--daemon status && + + # query will implicitly start the daemon. + # + # for test-script simplicity, we send a V1 timestamp rather than + # a V2 token. either way, the daemon response to any query contains + # a new V2 token. (the daemon may complain that we sent a V1 request, + # but this test case is only concerned with whether the daemon was + # implicitly started.) + + GIT_TRACE2_EVENT="$(pwd)/.git/trace" \ + test-tool -C test_implicit fsmonitor-client query --token 0 >actual && + nul_to_q actual.filtered && + grep "builtin:" actual.filtered && + + # confirm that a daemon was started in the background. + # + # since the mechanism for starting the background daemon is platform + # dependent, just confirm that the foreground command received a + # response from the daemon. + + have_t2_data_event fsm_client query/response-length <.git/trace && + + git -C test_implicit fsmonitor--daemon status && + git -C test_implicit fsmonitor--daemon stop && + test_must_fail git -C test_implicit fsmonitor--daemon status +' + +test_expect_success 'implicit daemon stop (delete .git)' ' + test_when_finished "stop_daemon_delete_repo test_implicit_1" && + + git init test_implicit_1 && + + start_daemon test_implicit_1 && + + # deleting the .git directory will implicitly stop the daemon. + rm -rf test_implicit_1/.git && + + # [1] Create an empty .git directory so that the following Git + # command will stay relative to the `-C` directory. + # + # Without this, the Git command will override the requested + # -C argument and crawl out to the containing Git source tree. + # This would make the test result dependent upon whether we + # were using fsmonitor on our development worktree. + # + sleep 1 && + mkdir test_implicit_1/.git && + + test_must_fail git -C test_implicit_1 fsmonitor--daemon status +' + +test_expect_success 'implicit daemon stop (rename .git)' ' + test_when_finished "stop_daemon_delete_repo test_implicit_2" && + + git init test_implicit_2 && + + start_daemon test_implicit_2 && + + # renaming the .git directory will implicitly stop the daemon. + mv test_implicit_2/.git test_implicit_2/.xxx && + + # See [1] above. + # + sleep 1 && + mkdir test_implicit_2/.git && + + test_must_fail git -C test_implicit_2 fsmonitor--daemon status +' + +test_expect_success 'cannot start multiple daemons' ' + test_when_finished "stop_daemon_delete_repo test_multiple" && + + git init test_multiple && + + start_daemon test_multiple && + + test_must_fail git -C test_multiple fsmonitor--daemon start 2>actual && + grep "fsmonitor--daemon is already running" actual && + + git -C test_multiple fsmonitor--daemon stop && + test_must_fail git -C test_multiple fsmonitor--daemon status +' + +# These tests use the main repo in the trash directory + +test_expect_success 'setup' ' + >tracked && + >modified && + >delete && + >rename && + mkdir dir1 && + >dir1/tracked && + >dir1/modified && + >dir1/delete && + >dir1/rename && + mkdir dir2 && + >dir2/tracked && + >dir2/modified && + >dir2/delete && + >dir2/rename && + mkdir dirtorename && + >dirtorename/a && + >dirtorename/b && + + cat >.gitignore <<-\EOF && + .gitignore + expect* + actual* + EOF + + git -c core.useBuiltinFSMonitor= add . && + test_tick && + git -c core.useBuiltinFSMonitor= commit -m initial && + + git config core.useBuiltinFSMonitor true +' + +# The test already explicitly stopped (or tried to stop) the daemon. +# This is here in case something else fails first. +# +redundant_stop_daemon () { + git fsmonitor--daemon stop + return 0 +} + +test_expect_success 'update-index implicitly starts daemon' ' + test_when_finished redundant_stop_daemon && + + test_must_fail git fsmonitor--daemon status && + + GIT_TRACE2_EVENT="$(pwd)/.git/trace_implicit_1" \ + git update-index --fsmonitor && + + git fsmonitor--daemon status && + test_might_fail git fsmonitor--daemon stop && + + # Confirm that the trace2 log contains a record of the + # daemon starting. + test_subcommand git fsmonitor--daemon start <.git/trace_implicit_1 +' + +test_expect_success 'status implicitly starts daemon' ' + test_when_finished redundant_stop_daemon && + + test_must_fail git fsmonitor--daemon status && + + GIT_TRACE2_EVENT="$(pwd)/.git/trace_implicit_2" \ + git status >actual && + + git fsmonitor--daemon status && + test_might_fail git fsmonitor--daemon stop && + + # Confirm that the trace2 log contains a record of the + # daemon starting. + test_subcommand git fsmonitor--daemon start <.git/trace_implicit_2 +' + +edit_files() { + echo 1 >modified + echo 2 >dir1/modified + echo 3 >dir2/modified + >dir1/untracked +} + +delete_files() { + rm -f delete + rm -f dir1/delete + rm -f dir2/delete +} + +create_files() { + echo 1 >new + echo 2 >dir1/new + echo 3 >dir2/new +} + +rename_files() { + mv rename renamed + mv dir1/rename dir1/renamed + mv dir2/rename dir2/renamed +} + +file_to_directory() { + rm -f delete + mkdir delete + echo 1 >delete/new +} + +directory_to_file() { + rm -rf dir1 + echo 1 >dir1 +} + +verify_status() { + git status >actual && + GIT_INDEX_FILE=.git/fresh-index git read-tree master && + GIT_INDEX_FILE=.git/fresh-index git -c core.useBuiltinFSMonitor= status >expect && + test_cmp expect actual && + echo HELLO AFTER && + cat .git/trace && + echo HELLO AFTER +} + +# The next few test cases confirm that our fsmonitor daemon sees each type +# of OS filesystem notification that we care about. At this layer we just +# ensure we are getting the OS notifications and do not try to confirm what +# is reported by `git status`. +# +# We run a simple query after modifying the filesystem just to introduce +# a bit of a delay so that the trace logging from the daemon has time to +# get flushed to disk. +# +# We `reset` and `clean` at the bottom of each test (and before stopping the +# daemon) because these commands might implicitly restart the daemon. + +clean_up_repo_and_stop_daemon () { + git reset --hard HEAD + git clean -fd + git fsmonitor--daemon stop + rm -f .git/trace +} + +test_expect_success 'edit some files' ' + test_when_finished clean_up_repo_and_stop_daemon && + + ( + GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && + export GIT_TRACE_FSMONITOR && + + start_daemon + ) && + + edit_files && + + test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && + + grep "^event: dir1/modified$" .git/trace && + grep "^event: dir2/modified$" .git/trace && + grep "^event: modified$" .git/trace && + grep "^event: dir1/untracked$" .git/trace +' + +test_expect_success 'create some files' ' + test_when_finished clean_up_repo_and_stop_daemon && + + ( + GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && + export GIT_TRACE_FSMONITOR && + + start_daemon + ) && + + create_files && + + test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && + + grep "^event: dir1/new$" .git/trace && + grep "^event: dir2/new$" .git/trace && + grep "^event: new$" .git/trace +' + +test_expect_success 'delete some files' ' + test_when_finished clean_up_repo_and_stop_daemon && + + ( + GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && + export GIT_TRACE_FSMONITOR && + + start_daemon + ) && + + delete_files && + + test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && + + grep "^event: dir1/delete$" .git/trace && + grep "^event: dir2/delete$" .git/trace && + grep "^event: delete$" .git/trace +' + +test_expect_success 'rename some files' ' + test_when_finished clean_up_repo_and_stop_daemon && + + ( + GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && + export GIT_TRACE_FSMONITOR && + + start_daemon + ) && + + rename_files && + + test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && + + grep "^event: dir1/rename$" .git/trace && + grep "^event: dir2/rename$" .git/trace && + grep "^event: rename$" .git/trace && + grep "^event: dir1/renamed$" .git/trace && + grep "^event: dir2/renamed$" .git/trace && + grep "^event: renamed$" .git/trace +' + +test_expect_success 'rename directory' ' + test_when_finished clean_up_repo_and_stop_daemon && + + ( + GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && + export GIT_TRACE_FSMONITOR && + + start_daemon + ) && + + mv dirtorename dirrenamed && + + test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && + + grep "^event: dirtorename/*$" .git/trace && + grep "^event: dirrenamed/*$" .git/trace +' + +test_expect_success 'file changes to directory' ' + test_when_finished clean_up_repo_and_stop_daemon && + + ( + GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && + export GIT_TRACE_FSMONITOR && + + start_daemon + ) && + + file_to_directory && + + test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && + + grep "^event: delete$" .git/trace && + grep "^event: delete/new$" .git/trace +' + +test_expect_success 'directory changes to a file' ' + test_when_finished clean_up_repo_and_stop_daemon && + + ( + GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && + export GIT_TRACE_FSMONITOR && + + start_daemon + ) && + + directory_to_file && + + test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && + + grep "^event: dir1$" .git/trace +' + +# The next few test cases exercise the token-resync code. When filesystem +# drops events (because of filesystem velocity or because the daemon isn't +# polling fast enough), we need to discard the cached data (relative to the +# current token) and start collecting events under a new token. +# +# the 'test-tool fsmonitor-client flush' command can be used to send a +# "flush" message to a running daemon and ask it to do a flush/resync. + +test_expect_success 'flush cached data' ' + test_when_finished "stop_daemon_delete_repo test_flush" && + + git init test_flush && + + ( + GIT_TEST_FSMONITOR_TOKEN=true && + export GIT_TEST_FSMONITOR_TOKEN && + + GIT_TRACE_FSMONITOR="$(pwd)/.git/trace_daemon" && + export GIT_TRACE_FSMONITOR && + + start_daemon test_flush + ) && + + # The daemon should have an initial token with no events in _0 and + # then a few (probably platform-specific number of) events in _1. + # These should both have the same . + + test-tool -C test_flush fsmonitor-client query --token "builtin:test_00000001:0" >actual_0 && + nul_to_q actual_q0 && + + touch test_flush/file_1 && + touch test_flush/file_2 && + + test-tool -C test_flush fsmonitor-client query --token "builtin:test_00000001:0" >actual_1 && + nul_to_q actual_q1 && + + grep "file_1" actual_q1 && + + # Force a flush. This will change the , reset the , and + # flush the file data. Then create some events and ensure that the file + # again appears in the cache. It should have the new . + + test-tool -C test_flush fsmonitor-client flush >flush_0 && + nul_to_q flush_q0 && + grep "^builtin:test_00000002:0Q/Q$" flush_q0 && + + test-tool -C test_flush fsmonitor-client query --token "builtin:test_00000002:0" >actual_2 && + nul_to_q actual_q2 && + + grep "^builtin:test_00000002:0Q$" actual_q2 && + + touch test_flush/file_3 && + + test-tool -C test_flush fsmonitor-client query --token "builtin:test_00000002:0" >actual_3 && + nul_to_q actual_q3 && + + grep "file_3" actual_q3 +' + +# The next few test cases create repos where the .git directory is NOT +# inside the one of the working directory. That is, where .git is a file +# that points to a directory elsewhere. This happens for submodules and +# non-primary worktrees. + +test_expect_success 'setup worktree base' ' + git init wt-base && + echo 1 >wt-base/file1 && + git -C wt-base add file1 && + git -C wt-base commit -m "c1" +' + +test_expect_success 'worktree with .git file' ' + git -C wt-base worktree add ../wt-secondary && + + ( + GIT_TRACE2_PERF="$(pwd)/trace2_wt_secondary" && + export GIT_TRACE2_PERF && + + GIT_TRACE_FSMONITOR="$(pwd)/trace_wt_secondary" && + export GIT_TRACE_FSMONITOR && + + start_daemon wt-secondary + ) && + + git -C wt-secondary fsmonitor--daemon stop && + test_must_fail git -C wt-secondary fsmonitor--daemon status +' + +# NEEDSWORK: Repeat one of the "edit" tests on wt-secondary and +# confirm that we get the same events and behavior -- that is, that +# fsmonitor--daemon correctly watches BOTH the working directory and +# the external GITDIR directory and behaves the same as when ".git" +# is a directory inside the working directory. + +test_expect_success 'cleanup worktrees' ' + stop_daemon_delete_repo wt-secondary && + stop_daemon_delete_repo wt-base +' + +test_done From 2ba3e3537f2d615dbe3e312140a4402836c7d576 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 21 Apr 2021 09:03:24 -0400 Subject: [PATCH 077/110] t/perf: avoid copying builtin fsmonitor files into test repo Do not copy any of the various fsmonitor--daemon files from the .git directory of the (GIT_PREF_REPO or GIT_PERF_LARGE_REPO) source repo into the test's trash directory. When perf tests start, they copy the contents of the source repo into the test's trash directory. If fsmonitor is running in the source repo, there may be control files, such as the IPC socket and/or fsmonitor cookie files. These should not be copied into the test repo. Unix domain sockets cannot be copied in the manner used by the test setup, so if present, the test setup fails. Cookie files are harmless, but we should avoid them. The builtin fsmonitor keeps all such control files/sockets in .git/fsmonitor--daemon*, so it is simple to exclude them. Signed-off-by: Jeff Hostetler --- t/perf/perf-lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/perf/perf-lib.sh b/t/perf/perf-lib.sh index f5ed092ee591ca..dc29df12fd5fda 100644 --- a/t/perf/perf-lib.sh +++ b/t/perf/perf-lib.sh @@ -74,7 +74,7 @@ test_perf_copy_repo_contents () { for stuff in "$1"/* do case "$stuff" in - */objects|*/hooks|*/config|*/commondir|*/gitdir|*/worktrees) + */objects|*/hooks|*/config|*/commondir|*/gitdir|*/worktrees|*/fsmonitor--daemon*) ;; *) cp -R "$stuff" "$repo/.git/" || exit 1 From 3b617e8862450ede938cb777c75691c8dccb919a Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 20 Jul 2021 13:41:35 -0400 Subject: [PATCH 078/110] t/helper/test-chmtime: skip directories on Windows Teach `test-tool.exe chmtime` to ignore errors when setting the mtime on a directory on Windows. NEEDSWORK: The Windows version of `utime()` (aka `mingw_utime()`) does not properly handle directories because it uses `_wopen()`. It should be converted to using `CreateFileW()` and backup semantics at a minimum. Since I'm already in the middle of a large patch series, I did not want to destabilize other callers of `utime()` right now. The problem has only been observed in the t/perf/p7519 test when the test repo contains an empty directory on disk. Signed-off-by: Jeff Hostetler --- t/helper/test-chmtime.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/t/helper/test-chmtime.c b/t/helper/test-chmtime.c index 524b55ca496cab..dc28890a183f1d 100644 --- a/t/helper/test-chmtime.c +++ b/t/helper/test-chmtime.c @@ -134,6 +134,21 @@ int cmd__chmtime(int argc, const char **argv) } if (utb.modtime != sb.st_mtime && utime(argv[i], &utb) < 0) { +#ifdef GIT_WINDOWS_NATIVE + if (S_ISDIR(sb.st_mode)) { + /* + * NEEDSWORK: The Windows version of `utime()` + * (aka `mingw_utime()`) does not correctly + * handle directory arguments, since it uses + * `_wopen()`. Ignore it for now since this + * is just a test. + */ + fprintf(stderr, + ("Failed to modify time on directory %s. " + "Skipping\n"), argv[i]); + continue; + } +#endif fprintf(stderr, "Failed to modify time on %s: %s\n", argv[i], strerror(errno)); return 1; From f8013f4cb24b327dff43d4d624c6485fb400d6cf Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 20 Jul 2021 13:49:48 -0400 Subject: [PATCH 079/110] t/perf/p7519: speed up test on Windows Change p7519 to use `test_seq` and `xargs` rather than a `for` loop to touch thousands of files. This takes minutes off of test runs on Windows because of process creation overhead. Signed-off-by: Jeff Hostetler --- t/perf/p7519-fsmonitor.sh | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/t/perf/p7519-fsmonitor.sh b/t/perf/p7519-fsmonitor.sh index 5eb5044a103cab..171644ffc90761 100755 --- a/t/perf/p7519-fsmonitor.sh +++ b/t/perf/p7519-fsmonitor.sh @@ -98,6 +98,13 @@ trace_stop() { fi } +touch_files() { + n=$1 + d="$n"_files + + (cd $d ; test_seq 1 $n | xargs touch ) +} + test_expect_success "one time repo setup" ' # set untrackedCache depending on the environment if test -n "$GIT_PERF_7519_UNTRACKED_CACHE" @@ -119,10 +126,11 @@ test_expect_success "one time repo setup" ' fi && mkdir 1_file 10_files 100_files 1000_files 10000_files && - for i in $(test_seq 1 10); do touch 10_files/$i; done && - for i in $(test_seq 1 100); do touch 100_files/$i; done && - for i in $(test_seq 1 1000); do touch 1000_files/$i; done && - for i in $(test_seq 1 10000); do touch 10000_files/$i; done && + touch_files 1 && + touch_files 10 && + touch_files 100 && + touch_files 1000 && + touch_files 10000 && git add 1_file 10_files 100_files 1000_files 10000_files && git commit -qm "Add files" && @@ -199,15 +207,15 @@ test_fsmonitor_suite() { # Update the mtimes on upto 100k files to make status think # that they are dirty. For simplicity, omit any files with - # LFs (i.e. anything that ls-files thinks it needs to dquote). - # Then fully backslash-quote the paths to capture any - # whitespace so that they pass thru xargs properly. + # LFs (i.e. anything that ls-files thinks it needs to dquote) + # and any files with whitespace so that they pass thru xargs + # properly. # test_perf_w_drop_caches "status (dirty) ($DESC)" ' git ls-files | \ head -100000 | \ grep -v \" | \ - sed '\''s/\(.\)/\\\1/g'\'' | \ + egrep -v " ." | \ xargs test-tool chmtime -300 && git status ' From 065492de0a948aefc3a5a7cf4e4b40f6dd7316de Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 15 Jan 2021 15:38:42 -0500 Subject: [PATCH 080/110] t/perf/p7519: add fsmonitor--daemon test cases Repeat all of the fsmonitor perf tests using `git fsmonitor--daemon` and the "Simple IPC" interface. Signed-off-by: Jeff Hostetler --- t/perf/p7519-fsmonitor.sh | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/t/perf/p7519-fsmonitor.sh b/t/perf/p7519-fsmonitor.sh index 171644ffc90761..e70252ed65ab70 100755 --- a/t/perf/p7519-fsmonitor.sh +++ b/t/perf/p7519-fsmonitor.sh @@ -24,7 +24,8 @@ test_description="Test core.fsmonitor" # GIT_PERF_7519_SPLIT_INDEX: used to configure core.splitIndex # GIT_PERF_7519_FSMONITOR: used to configure core.fsMonitor. May be an # absolute path to an integration. May be a space delimited list of -# absolute paths to integrations. +# absolute paths to integrations. (This hook or list of hooks does not +# include the built-in fsmonitor--daemon.) # # The big win for using fsmonitor is the elimination of the need to scan the # working directory looking for changed and untracked files. If the file @@ -143,10 +144,16 @@ test_expect_success "one time repo setup" ' setup_for_fsmonitor() { # set INTEGRATION_SCRIPT depending on the environment - if test -n "$INTEGRATION_PATH" + if test -n "$USE_FSMONITOR_DAEMON" then + git config core.useBuiltinFSMonitor true && + INTEGRATION_SCRIPT=false + elif test -n "$INTEGRATION_PATH" + then + git config core.useBuiltinFSMonitor false && INTEGRATION_SCRIPT="$INTEGRATION_PATH" else + git config core.useBuiltinFSMonitor false && # # Choose integration script based on existence of Watchman. # Fall back to an empty integration script. @@ -182,7 +189,10 @@ test_perf_w_drop_caches () { } test_fsmonitor_suite() { - if test -n "$INTEGRATION_SCRIPT"; then + if test -n "$USE_FSMONITOR_DAEMON" + then + DESC="builtin fsmonitor--daemon" + elif test -n "$INTEGRATION_SCRIPT"; then DESC="fsmonitor=$(basename $INTEGRATION_SCRIPT)" else DESC="fsmonitor=disabled" @@ -293,4 +303,25 @@ test_expect_success "setup without fsmonitor" ' test_fsmonitor_suite trace_stop +# +# Run a full set of perf tests using the built-in fsmonitor--daemon. +# It does not use the Hook API, so it has a different setup. +# Explicitly start the daemon here and before we start client commands +# so that we can later add custom tracing. +# +if test_have_prereq FSMONITOR_DAEMON +then + USE_FSMONITOR_DAEMON=t + + trace_start fsmonitor--daemon--server + git fsmonitor--daemon start + + trace_start fsmonitor--daemon--client + test_expect_success "setup for fsmonitor--daemon" 'setup_for_fsmonitor' + test_fsmonitor_suite + + git fsmonitor--daemon stop + trace_stop +fi + test_done From 4c1167f1497fb0df33196af83ad473d0b7820d6e Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 18 Dec 2020 11:46:06 -0500 Subject: [PATCH 081/110] fsmonitor--daemon: periodically truncate list of modified files Teach fsmonitor--daemon to periodically truncate the list of modified files to save some memory. Clients will ask for the set of changes relative to a token that they found in the FSMN index extension in the index. (This token is like a point in time, but different). Clients will then update the index to contain the response token (so that subsequent commands will be relative to this new token). Therefore, the daemon can gradually truncate the in-memory list of changed paths as they become obsolete (older than the previous token). Since we may have multiple clients making concurrent requests with a skew of tokens and clients may be racing to the talk to the daemon, we lazily truncate the list. We introduce a 5 minute delay and truncate batches 5 minutes after they are considered obsolete. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 88 +++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 4c88171e06c76c..962b24569e1bc8 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -299,6 +299,75 @@ static void fsmonitor_batch__combine(struct fsmonitor_batch *batch_dest, batch_src->interned_paths[k]; } +/* + * To keep the batch list from growing unbounded in response to filesystem + * activity, we try to truncate old batches from the end of the list as + * they become irrelevant. + * + * We assume that the .git/index will be updated with the most recent token + * any time the index is updated. And future commands will only ask for + * recent changes *since* that new token. So as tokens advance into the + * future, older batch items will never be requested/needed. So we can + * truncate them without loss of functionality. + * + * However, multiple commands may be talking to the daemon concurrently + * or perform a slow command, so a little "token skew" is possible. + * Therefore, we want this to be a little bit lazy and have a generous + * delay. + * + * The current reader thread walked backwards in time from `token->batch_head` + * back to `batch_marker` somewhere in the middle of the batch list. + * + * Let's walk backwards in time from that marker an arbitrary delay + * and truncate the list there. Note that these timestamps are completely + * artificial (based on when we pinned the batch item) and not on any + * filesystem activity. + * + * Return the obsolete portion of the list after we have removed it from + * the official list so that the caller can free it after leaving the lock. + */ +#define MY_TIME_DELAY_SECONDS (5 * 60) /* seconds */ + +static struct fsmonitor_batch *with_lock__truncate_old_batches( + struct fsmonitor_daemon_state *state, + const struct fsmonitor_batch *batch_marker) +{ + /* assert current thread holding state->main_lock */ + + const struct fsmonitor_batch *batch; + struct fsmonitor_batch *remainder; + + if (!batch_marker) + return NULL; + + trace_printf_key(&trace_fsmonitor, "Truncate: mark (%"PRIu64",%"PRIu64")", + batch_marker->batch_seq_nr, + (uint64_t)batch_marker->pinned_time); + + for (batch = batch_marker; batch; batch = batch->next) { + time_t t; + + if (!batch->pinned_time) /* an overflow batch */ + continue; + + t = batch->pinned_time + MY_TIME_DELAY_SECONDS; + if (t > batch_marker->pinned_time) /* too close to marker */ + continue; + + goto truncate_past_here; + } + + return NULL; + +truncate_past_here: + state->current_token_data->batch_tail = (struct fsmonitor_batch *)batch; + + remainder = ((struct fsmonitor_batch *)batch)->next; + ((struct fsmonitor_batch *)batch)->next = NULL; + + return remainder; +} + static void fsmonitor_free_token_data(struct fsmonitor_token_data *token) { if (!token) @@ -412,6 +481,7 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, const char *p; const struct fsmonitor_batch *batch_head; const struct fsmonitor_batch *batch; + struct fsmonitor_batch *remainder = NULL; intmax_t count = 0, duplicates = 0; kh_str_t *shown; int hash_ret; @@ -641,11 +711,29 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, * that work. */ fsmonitor_free_token_data(token_data); + } else if (batch) { + /* + * We are holding the lock and are the only + * reader of the ref-counted portion of the + * list, so we get the honor of seeing if the + * list can be truncated to save memory. + * + * The main loop did not walk to the end of the + * list, so this batch is the first item in the + * batch-list that is older than the requested + * end-point sequence number. See if the tail + * end of the list is obsolete. + */ + remainder = with_lock__truncate_old_batches(state, + batch); } } pthread_mutex_unlock(&state->main_lock); + if (remainder) + fsmonitor_batch__free_list(remainder); + trace2_data_intmax("fsmonitor", the_repository, "response/length", total_response_len); trace2_data_intmax("fsmonitor", the_repository, "response/count/files", count); trace2_data_intmax("fsmonitor", the_repository, "response/count/duplicates", duplicates); From a511adef6d451f927ee427c3d9137df3121a395c Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 18 Dec 2020 12:11:29 -0500 Subject: [PATCH 082/110] fsmonitor--daemon: use a cookie file to sync with file system Teach fsmonitor--daemon client threads to create a cookie file inside the .git directory and then wait until FS events for the cookie are observed by the FS listener thread. This helps address the racy nature of file system events by blocking the client response until the kernel has drained any event backlog. This is especially important on MacOS where kernel events are only issued with a limited frequency. See the `latency` argument of `FSeventStreamCreate()`. The kernel only signals every `latency` seconds, but does not guarantee that the kernel queue is completely drained, so we may have to wait more than one interval. If we increase the frequency, the system is more likely to drop events. We avoid these issues by having each client thread create a unique cookie file and then wait until it is seen in the event stream. Co-authored-by: Kevin Willford Co-authored-by: Johannes Schindelin Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 228 +++++++++++++++++++++++++++++++++++- fsmonitor--daemon.h | 5 + 2 files changed, 232 insertions(+), 1 deletion(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 962b24569e1bc8..6011fe42ee0e5b 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -94,6 +94,149 @@ static int do_as_client__status(void) } } +enum fsmonitor_cookie_item_result { + FCIR_ERROR = -1, /* could not create cookie file ? */ + FCIR_INIT = 0, + FCIR_SEEN, + FCIR_ABORT, +}; + +struct fsmonitor_cookie_item { + struct hashmap_entry entry; + const char *name; + enum fsmonitor_cookie_item_result result; +}; + +static int cookies_cmp(const void *data, const struct hashmap_entry *he1, + const struct hashmap_entry *he2, const void *keydata) +{ + const struct fsmonitor_cookie_item *a = + container_of(he1, const struct fsmonitor_cookie_item, entry); + const struct fsmonitor_cookie_item *b = + container_of(he2, const struct fsmonitor_cookie_item, entry); + + return strcmp(a->name, keydata ? keydata : b->name); +} + +static enum fsmonitor_cookie_item_result with_lock__wait_for_cookie( + struct fsmonitor_daemon_state *state) +{ + /* assert current thread holding state->main_lock */ + + int fd; + struct fsmonitor_cookie_item *cookie; + struct strbuf cookie_pathname = STRBUF_INIT; + struct strbuf cookie_filename = STRBUF_INIT; + enum fsmonitor_cookie_item_result result; + int my_cookie_seq; + + CALLOC_ARRAY(cookie, 1); + + my_cookie_seq = state->cookie_seq++; + + strbuf_addf(&cookie_filename, "%i-%i", getpid(), my_cookie_seq); + + strbuf_addbuf(&cookie_pathname, &state->path_cookie_prefix); + strbuf_addbuf(&cookie_pathname, &cookie_filename); + + cookie->name = strbuf_detach(&cookie_filename, NULL); + cookie->result = FCIR_INIT; + hashmap_entry_init(&cookie->entry, strhash(cookie->name)); + + hashmap_add(&state->cookies, &cookie->entry); + + trace_printf_key(&trace_fsmonitor, "cookie-wait: '%s' '%s'", + cookie->name, cookie_pathname.buf); + + /* + * Create the cookie file on disk and then wait for a notification + * that the listener thread has seen it. + */ + fd = open(cookie_pathname.buf, O_WRONLY | O_CREAT | O_EXCL, 0600); + if (fd >= 0) { + close(fd); + unlink(cookie_pathname.buf); + + /* + * NEEDSWORK: This is an infinite wait (well, unless another + * thread sends us an abort). I'd like to change this to + * use `pthread_cond_timedwait()` and return an error/timeout + * and let the caller do the trivial response thing. + */ + while (cookie->result == FCIR_INIT) + pthread_cond_wait(&state->cookies_cond, + &state->main_lock); + } else { + error_errno(_("could not create fsmonitor cookie '%s'"), + cookie->name); + + cookie->result = FCIR_ERROR; + } + + hashmap_remove(&state->cookies, &cookie->entry, NULL); + + result = cookie->result; + + free((char*)cookie->name); + free(cookie); + strbuf_release(&cookie_pathname); + + return result; +} + +/* + * Mark these cookies as _SEEN and wake up the corresponding client threads. + */ +static void with_lock__mark_cookies_seen(struct fsmonitor_daemon_state *state, + const struct string_list *cookie_names) +{ + /* assert current thread holding state->main_lock */ + + int k; + int nr_seen = 0; + + for (k = 0; k < cookie_names->nr; k++) { + struct fsmonitor_cookie_item key; + struct fsmonitor_cookie_item *cookie; + + key.name = cookie_names->items[k].string; + hashmap_entry_init(&key.entry, strhash(key.name)); + + cookie = hashmap_get_entry(&state->cookies, &key, entry, NULL); + if (cookie) { + trace_printf_key(&trace_fsmonitor, "cookie-seen: '%s'", + cookie->name); + cookie->result = FCIR_SEEN; + nr_seen++; + } + } + + if (nr_seen) + pthread_cond_broadcast(&state->cookies_cond); +} + +/* + * Set _ABORT on all pending cookies and wake up all client threads. + */ +static void with_lock__abort_all_cookies(struct fsmonitor_daemon_state *state) +{ + /* assert current thread holding state->main_lock */ + + struct hashmap_iter iter; + struct fsmonitor_cookie_item *cookie; + int nr_aborted = 0; + + hashmap_for_each_entry(&state->cookies, &iter, cookie, entry) { + trace_printf_key(&trace_fsmonitor, "cookie-abort: '%s'", + cookie->name); + cookie->result = FCIR_ABORT; + nr_aborted++; + } + + if (nr_aborted) + pthread_cond_broadcast(&state->cookies_cond); +} + /* * Requests to and from a FSMonitor Protocol V2 provider use an opaque * "token" as a virtual timestamp. Clients can request a summary of all @@ -391,6 +534,9 @@ static void fsmonitor_free_token_data(struct fsmonitor_token_data *token) * We should create a new token and start fresh (as if we just * booted up). * + * [2] Some of those lost events may have been for cookie files. We + * should assume the worst and abort them rather letting them starve. + * * If there are no concurrent threads readering the current token data * series, we can free it now. Otherwise, let the last reader free * it. @@ -412,6 +558,8 @@ static void with_lock__do_force_resync(struct fsmonitor_daemon_state *state) state->current_token_data = new_one; fsmonitor_free_token_data(free_me); + + with_lock__abort_all_cookies(state); } void fsmonitor_force_resync(struct fsmonitor_daemon_state *state) @@ -487,6 +635,8 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, int hash_ret; int do_trivial = 0; int do_flush = 0; + int do_cookie = 0; + enum fsmonitor_cookie_item_result cookie_result; /* * We expect `command` to be of the form: @@ -547,6 +697,7 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, * We have a V2 valid token: * "builtin::" */ + do_cookie = 1; } } @@ -555,6 +706,30 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, if (!state->current_token_data) BUG("fsmonitor state does not have a current token"); + /* + * Write a cookie file inside the directory being watched in + * an effort to flush out existing filesystem events that we + * actually care about. Suspend this client thread until we + * see the filesystem events for this cookie file. + * + * Creating the cookie lets us guarantee that our FS listener + * thread has drained the kernel queue and we are caught up + * with the kernel. + * + * If we cannot create the cookie (or otherwise guarantee that + * we are caught up), we send a trivial response. We have to + * assume that there might be some very, very recent activity + * on the FS still in flight. + */ + if (do_cookie) { + cookie_result = with_lock__wait_for_cookie(state); + if (cookie_result != FCIR_SEEN) { + error(_("fsmonitor: cookie_result '%d' != SEEN"), + cookie_result); + do_trivial = 1; + } + } + if (do_flush) with_lock__do_force_resync(state); @@ -775,7 +950,9 @@ static int handle_client(void *data, return result; } -#define FSMONITOR_COOKIE_PREFIX ".fsmonitor-daemon-" +#define FSMONITOR_DIR "fsmonitor--daemon" +#define FSMONITOR_COOKIE_DIR "cookies" +#define FSMONITOR_COOKIE_PREFIX (FSMONITOR_DIR "/" FSMONITOR_COOKIE_DIR "/") enum fsmonitor_path_type fsmonitor_classify_path_workdir_relative( const char *rel) @@ -928,6 +1105,9 @@ void fsmonitor_publish(struct fsmonitor_daemon_state *state, } } + if (cookie_names->nr) + with_lock__mark_cookies_seen(state, cookie_names); + pthread_mutex_unlock(&state->main_lock); } @@ -1019,7 +1199,9 @@ static int fsmonitor_run_daemon(void) memset(&state, 0, sizeof(state)); + hashmap_init(&state.cookies, cookies_cmp, NULL, 0); pthread_mutex_init(&state.main_lock, NULL); + pthread_cond_init(&state.cookies_cond, NULL); state.error_code = 0; state.current_token_data = fsmonitor_new_token_data(); @@ -1044,6 +1226,44 @@ static int fsmonitor_run_daemon(void) state.nr_paths_watching = 2; } + /* + * We will write filesystem syncing cookie files into + * ///-. + * + * The extra layers of subdirectories here keep us from + * changing the mtime on ".git/" or ".git/foo/" when we create + * or delete cookie files. + * + * There have been problems with some IDEs that do a + * non-recursive watch of the ".git/" directory and run a + * series of commands any time something happens. + * + * For example, if we place our cookie files directly in + * ".git/" or ".git/foo/" then a `git status` (or similar + * command) from the IDE will cause a cookie file to be + * created in one of those dirs. This causes the mtime of + * those dirs to change. This triggers the IDE's watch + * notification. This triggers the IDE to run those commands + * again. And the process repeats and the machine never goes + * idle. + * + * Adding the extra layers of subdirectories prevents the + * mtime of ".git/" and ".git/foo" from changing when a + * cookie file is created. + */ + strbuf_init(&state.path_cookie_prefix, 0); + strbuf_addbuf(&state.path_cookie_prefix, &state.path_gitdir_watch); + + strbuf_addch(&state.path_cookie_prefix, '/'); + strbuf_addstr(&state.path_cookie_prefix, FSMONITOR_DIR); + mkdir(state.path_cookie_prefix.buf, 0777); + + strbuf_addch(&state.path_cookie_prefix, '/'); + strbuf_addstr(&state.path_cookie_prefix, FSMONITOR_COOKIE_DIR); + mkdir(state.path_cookie_prefix.buf, 0777); + + strbuf_addch(&state.path_cookie_prefix, '/'); + /* * Confirm that we can create platform-specific resources for the * filesystem listener before we bother starting all the threads. @@ -1056,6 +1276,7 @@ static int fsmonitor_run_daemon(void) err = fsmonitor_run_daemon_1(&state); done: + pthread_cond_destroy(&state.cookies_cond); pthread_mutex_destroy(&state.main_lock); fsm_listen__dtor(&state); @@ -1063,6 +1284,11 @@ static int fsmonitor_run_daemon(void) strbuf_release(&state.path_worktree_watch); strbuf_release(&state.path_gitdir_watch); + strbuf_release(&state.path_cookie_prefix); + + /* + * NEEDSWORK: Consider "rm -rf /" + */ return err; } diff --git a/fsmonitor--daemon.h b/fsmonitor--daemon.h index 20a815d80f8081..c16ef0956883da 100644 --- a/fsmonitor--daemon.h +++ b/fsmonitor--daemon.h @@ -45,6 +45,11 @@ struct fsmonitor_daemon_state { struct fsmonitor_token_data *current_token_data; + struct strbuf path_cookie_prefix; + pthread_cond_t cookies_cond; + int cookie_seq; + struct hashmap cookies; + int error_code; struct fsmonitor_daemon_backend_data *backend_data; From 67f2c1df7982f748225bb373436306ab0a13fb3b Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 19 May 2021 15:42:21 -0400 Subject: [PATCH 083/110] fsmonitor: force update index after large responses Set the `FSMONITOR_CHANGED` bit on `istate->cache_changed` when FSMonitor returns a very large repsonse to ensure that the index is written to disk. Normally, when the FSMonitor response includes a tracked file, the index is always updated. Similarly, the index might be updated when the response alters the untracked-cache (when enabled). However, in cases where neither of those cause the index to be considered changed, the FSMonitor response is wasted. Subsequent Git commands will make requests with the same token and receive the same response. If that response is very large, performance may suffer. It would be more efficient to force update the index now (and the token in the index extension) in order to reduce the size of the response received by future commands. This was observed on Windows after a large checkout. On Windows, the kernel emits events for the files that are changed as they are changed. However, it might delay events for the containing directories until the system is more idle (or someone scans the directory (so it seems)). The first status following a checkout would get the list of files. The subsequent status commands would get the list of directories as the events trickled out. But they would never catch up because the token was not advanced because the index wasn't updated. This list of directories caused `wt_status_collect_untracked()` to unnecessarily spend time actually scanning them during each command. Signed-off-by: Jeff Hostetler --- fsmonitor.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/fsmonitor.c b/fsmonitor.c index 695fb0ce4e7d75..3a63cda58f73f0 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -233,6 +233,45 @@ static void fsmonitor_refresh_callback(struct index_state *istate, char *name) untracked_cache_invalidate_path(istate, name, 0); } +/* + * The number of pathnames that we need to receive from FSMonitor + * before we force the index to be updated. + * + * Note that any pathname within the set of received paths MAY cause + * cache-entry or istate flag bits to be updated and thus cause the + * index to be updated on disk. + * + * However, the response may contain many paths (such as ignored + * paths) that will not update any flag bits. And thus not force the + * index to be updated. (This is fine and normal.) It also means + * that the token will not be updated in the FSMonitor index + * extension. So the next Git command will find the same token in the + * index, make the same token-relative request, and receive the same + * response (plus any newly changed paths). If this response is large + * (and continues to grow), performance could be impacted. + * + * For example, if the user runs a build and it writes 100K object + * files but doesn't modify any source files, the index would not need + * to be updated. The FSMonitor response (after the build and + * relative to a pre-build token) might be 5MB. Each subsequent Git + * command will receive that same 100K/5MB response until something + * causes the index to be updated. And `refresh_fsmonitor()` will + * have to iterate over those 100K paths each time. + * + * Performance could be improved if we optionally force update the + * index after a very large response and get an updated token into + * the FSMonitor index extension. This should allow subsequent + * commands to get smaller and more current responses. + * + * The value chosen here does not need to be precise. The index + * will be updated automatically the first time the user touches + * a tracked file and causes a command like `git status` to + * update an mtime to be updated and/or set a flag bit. + * + * NEEDSWORK: Does this need to be a config value? + */ +static int fsmonitor_force_update_threshold = 100; + void refresh_fsmonitor(struct index_state *istate) { struct strbuf query_result = STRBUF_INIT; @@ -370,19 +409,28 @@ void refresh_fsmonitor(struct index_state *istate) * * This updates both the cache-entries and the untracked-cache. */ + int count = 0; + buf = query_result.buf; for (i = bol; i < query_result.len; i++) { if (buf[i] != '\0') continue; fsmonitor_refresh_callback(istate, buf + bol); bol = i + 1; + count++; } - if (bol < query_result.len) + if (bol < query_result.len) { fsmonitor_refresh_callback(istate, buf + bol); + count++; + } /* Now mark the untracked cache for fsmonitor usage */ if (istate->untracked) istate->untracked->use_fsmonitor = 1; + + if (count > fsmonitor_force_update_threshold) + istate->cache_changed |= FSMONITOR_CHANGED; + } else { /* * We received a trivial response, so invalidate everything. From 4597bc9a8afdcf4778e1597716848bf9a4b88fb9 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 25 Jun 2021 18:04:56 -0400 Subject: [PATCH 084/110] compat/fsmonitor/fsm-listen-win32: handle shortnames Teach FSMonitor daemon on Windows to recognize shortname paths as aliases of normal longname paths. FSMonitor clients, such as `git status`, should receive the longname spelling of changed files (when possible). Sometimes we receive FS events using the shortname, such as when a CMD shell runs "RENAME GIT~1 FOO" or "RMDIR GIT~1". The FS notification arrives using whatever combination of long and shortnames were used by the other process. (Shortnames do seem to be case normalized, however.) Use Windows GetLongPathNameW() to try to map the pathname spelling in the notification event into the normalized longname spelling. (This can fail if the file/directory is deleted, moved, or renamed, because we are asking the FS for the mapping in response to the event and after it has already happened, but we try.) Special case the shortname spelling of ".git" to avoid under-reporting these events. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-listen-win32.c | 359 +++++++++++++++++++++++----- t/t7527-builtin-fsmonitor.sh | 65 +++++ 2 files changed, 370 insertions(+), 54 deletions(-) diff --git a/compat/fsmonitor/fsm-listen-win32.c b/compat/fsmonitor/fsm-listen-win32.c index c2d11acbc1ef7d..eb407b0748cf0f 100644 --- a/compat/fsmonitor/fsm-listen-win32.c +++ b/compat/fsmonitor/fsm-listen-win32.c @@ -25,6 +25,9 @@ struct one_watch DWORD count; struct strbuf path; + wchar_t wpath_longname[MAX_PATH + 1]; + DWORD wpath_longname_len; + HANDLE hDir; HANDLE hEvent; OVERLAPPED overlapped; @@ -34,6 +37,21 @@ struct one_watch * need to later call GetOverlappedResult() and possibly CancelIoEx(). */ BOOL is_active; + + /* + * Are shortnames enabled on the containing drive? This is + * always true for "C:/" drives and usually never true for + * other drives. + * + * We only set this for the worktree because we only need to + * convert shortname paths to longname paths for items we send + * to clients. (We don't care about shortname expansion for + * paths inside a GITDIR because we never send them to + * clients.) + */ + BOOL has_shortnames; + BOOL has_tilda; + wchar_t dotgit_shortname[16]; /* for 8.3 name */ }; struct fsmonitor_daemon_backend_data @@ -51,17 +69,18 @@ struct fsmonitor_daemon_backend_data }; /* - * Convert the WCHAR path from the notification into UTF8 and - * then normalize it. + * Convert the WCHAR path from the event into UTF8 and normalize it. + * + * `wpath_len` is in WCHARS not bytes. */ -static int normalize_path_in_utf8(FILE_NOTIFY_INFORMATION *info, +static int normalize_path_in_utf8(wchar_t *wpath, DWORD wpath_len, struct strbuf *normalized_path) { int reserve; int len = 0; strbuf_reset(normalized_path); - if (!info->FileNameLength) + if (!wpath_len) goto normalize; /* @@ -70,12 +89,12 @@ static int normalize_path_in_utf8(FILE_NOTIFY_INFORMATION *info, * sequence of 2 UTF8 characters. That should let us * avoid ERROR_INSUFFICIENT_BUFFER 99.9+% of the time. */ - reserve = info->FileNameLength + 1; + reserve = 2 * wpath_len + 1; strbuf_grow(normalized_path, reserve); for (;;) { - len = WideCharToMultiByte(CP_UTF8, 0, info->FileName, - info->FileNameLength / sizeof(WCHAR), + len = WideCharToMultiByte(CP_UTF8, 0, + wpath, wpath_len, normalized_path->buf, strbuf_avail(normalized_path) - 1, NULL, NULL); @@ -83,9 +102,7 @@ static int normalize_path_in_utf8(FILE_NOTIFY_INFORMATION *info, goto normalize; if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) { error("[GLE %ld] could not convert path to UTF-8: '%.*ls'", - GetLastError(), - (int)(info->FileNameLength / sizeof(WCHAR)), - info->FileName); + GetLastError(), (int)wpath_len, wpath); return -1; } @@ -98,6 +115,148 @@ static int normalize_path_in_utf8(FILE_NOTIFY_INFORMATION *info, return strbuf_normalize_path(normalized_path); } +/* + * See if the worktree root directory has shortnames enabled. + * This will help us decide if we need to do an expensive shortname + * to longname conversion on every notification event. + * + * We do not want to create a file to test this, so we assume that the + * root directory contains a ".git" file or directory. (Out caller + * only calls us for the worktree root, so this should be fine.) + * + * Remember the spelling of the shortname for ".git" if it exists. + */ +static void check_for_shortnames(struct one_watch *watch) +{ + wchar_t buf_in[MAX_PATH + 1]; + wchar_t buf_out[MAX_PATH + 1]; + wchar_t *last_slash = NULL; + wchar_t *last_bslash = NULL; + wchar_t *last; + + /* build L"/.git" */ + wcscpy(buf_in, watch->wpath_longname); + wcscpy(buf_in + watch->wpath_longname_len, L".git"); + + if (!GetShortPathNameW(buf_in, buf_out, MAX_PATH)) + return; + + last_slash = wcsrchr(buf_out, L'/'); + last_bslash = wcsrchr(buf_out, L'\\'); + if (last_slash > last_bslash) + last = last_slash + 1; + else if (last_bslash) + last = last_bslash + 1; + else + last = buf_out; + + if (!wcscmp(last, L".git")) + return; + + watch->has_shortnames = 1; + wcsncpy(watch->dotgit_shortname, last, + ARRAY_SIZE(watch->dotgit_shortname)); + + /* + * The shortname for ".git" is usually of the form "GIT~1", so + * we should be able to avoid shortname to longname mapping on + * every notification event if the source string does not + * contain a "~". + * + * However, the documentation for GetLongPathNameW() says + * that there are filesystems that don't follow that pattern + * and warns against this optimization. + * + * Lets test this. + */ + if (wcschr(watch->dotgit_shortname, L'~')) + watch->has_tilda = 1; +} + +enum get_relative_result { + GRR_NO_CONVERSION_NEEDED, + GRR_HAVE_CONVERSION, + GRR_SHUTDOWN, +}; + +/* + * Info notification paths are relative to the root of the watch. + * If our CWD is still at the root, then we can use relative paths + * to convert from shortnames to longnames. If our process has a + * different CWD, then we need to construct an absolute path, do + * the conversion, and then return the root-relative portion. + * + * We use the longname form of the root as our basis and assume that + * it already has a trailing slash. + * + * `wpath_len` is in WCHARS not bytes. + */ +static enum get_relative_result get_relative_longname( + struct one_watch *watch, + const wchar_t *wpath, DWORD wpath_len, + wchar_t *wpath_longname) +{ + wchar_t buf_in[2 * MAX_PATH + 1]; + wchar_t buf_out[MAX_PATH + 1]; + DWORD root_len; + + /* Build L"/" */ + root_len = watch->wpath_longname_len; + wcsncpy(buf_in, watch->wpath_longname, root_len); + wcsncpy(buf_in + root_len, wpath, wpath_len); + buf_in[root_len + wpath_len] = 0; + + /* + * We don't actually know if the source pathname is a + * shortname or a longname. This routine allows either to be + * given as input. + */ + if (!GetLongPathNameW(buf_in, buf_out, MAX_PATH)) { + /* + * The shortname to longname conversion can fail for + * various reasons, for example if the file has been + * deleted. (That is, if we just received a + * delete-file notification event and the file is + * already gone, we can't ask the file system to + * lookup the longname for it. Likewise, for moves + * and renames where we are given the old name.) + * + * NEEDSWORK: Since deleting or moving a file or + * directory by shortname is rather obscure, I'm going + * ignore the failure and ask the caller to report the + * original relative path. This seemds kinder than + * failing here and forcing a resync. + */ + return GRR_NO_CONVERSION_NEEDED; + } + + if (!wcscmp(buf_in, buf_out)) { + /* + * The path does not have a shortname alias. + */ + return GRR_NO_CONVERSION_NEEDED; + } + + if (wcsncmp(buf_in, buf_out, root_len)) { + /* + * The spelling of the root directory portion of the computed + * longname has changed. This should not happen. Basically, + * it means that we don't know where (without recomputing the + * longname of just the root directory) to split out the + * relative path. Since this should not happen, I'm just + * going to let this fail and force a shutdown (because all + * subsequent events are probably going to see the same + * mismatch). + */ + return GRR_SHUTDOWN; + } + + /* Return the worktree root-relative portion of the longname. */ + + wcscpy(wpath_longname, buf_out + root_len); + return GRR_HAVE_CONVERSION; +} + void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) { SetEvent(state->backend_data->hListener[LISTENER_SHUTDOWN]); @@ -111,7 +270,9 @@ static struct one_watch *create_watch(struct fsmonitor_daemon_state *state, DWORD share_mode = FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE; HANDLE hDir; - wchar_t wpath[MAX_PATH]; + DWORD len_longname; + wchar_t wpath[MAX_PATH + 1]; + wchar_t wpath_longname[MAX_PATH + 1]; if (xutftowcs_path(wpath, path) < 0) { error(_("could not convert to wide characters: '%s'"), path); @@ -128,6 +289,20 @@ static struct one_watch *create_watch(struct fsmonitor_daemon_state *state, return NULL; } + if (!GetLongPathNameW(wpath, wpath_longname, MAX_PATH)) { + error(_("[GLE %ld] could not get longname of '%s'"), + GetLastError(), path); + CloseHandle(hDir); + return NULL; + } + + len_longname = wcslen(wpath_longname); + if (wpath_longname[len_longname - 1] != L'/' && + wpath_longname[len_longname - 1] != L'\\') { + wpath_longname[len_longname++] = L'/'; + wpath_longname[len_longname] = 0; + } + CALLOC_ARRAY(watch, 1); watch->buf_len = sizeof(watch->buffer); /* assume full MAX_RDCW_BUF */ @@ -135,6 +310,9 @@ static struct one_watch *create_watch(struct fsmonitor_daemon_state *state, strbuf_init(&watch->path, 0); strbuf_addstr(&watch->path, path); + wcscpy(watch->wpath_longname, wpath_longname); + watch->wpath_longname_len = len_longname; + watch->hDir = hDir; watch->hEvent = CreateEvent(NULL, TRUE, FALSE, NULL); @@ -258,6 +436,62 @@ static void cancel_rdcw_watch(struct one_watch *watch) watch->is_active = FALSE; } +/* + * Process a single relative pathname event. + * Return 1 if we should shutdown. + */ +static int process_1_worktree_event( + struct string_list *cookie_list, + struct fsmonitor_batch **batch, + const struct strbuf *path, + enum fsmonitor_path_type t, + DWORD info_action) +{ + const char *slash; + + switch (t) { + case IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX: + /* special case cookie files within .git */ + + /* Use just the filename of the cookie file. */ + slash = find_last_dir_sep(path->buf); + string_list_append(cookie_list, + slash ? slash + 1 : path->buf); + break; + + case IS_INSIDE_DOT_GIT: + /* ignore everything inside of "/.git/" */ + break; + + case IS_DOT_GIT: + /* "/.git" was deleted (or renamed away) */ + if ((info_action == FILE_ACTION_REMOVED) || + (info_action == FILE_ACTION_RENAMED_OLD_NAME)) { + trace2_data_string("fsmonitor", NULL, + "fsm-listen/dotgit", + "removed"); + return 1; + } + break; + + case IS_WORKDIR_PATH: + /* queue normal pathname */ + if (!*batch) + *batch = fsmonitor_batch__new(); + fsmonitor_batch__add_path(*batch, path->buf); + break; + + case IS_GITDIR: + case IS_INSIDE_GITDIR: + case IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX: + default: + BUG("unexpected path classification '%d' for '%s'", + t, path->buf); + } + + return 0; +} + /* * Process filesystem events that happen anywhere (recursively) under the * root directory. For a normal working directory, this includes @@ -274,6 +508,7 @@ static int process_worktree_events(struct fsmonitor_daemon_state *state) struct string_list cookie_list = STRING_LIST_INIT_DUP; struct fsmonitor_batch *batch = NULL; const char *p = watch->buffer; + wchar_t wpath_longname[MAX_PATH + 1]; /* * If the kernel gets more events than will fit in the kernel @@ -306,54 +541,63 @@ static int process_worktree_events(struct fsmonitor_daemon_state *state) */ for (;;) { FILE_NOTIFY_INFORMATION *info = (void *)p; - const char *slash; + wchar_t *wpath = info->FileName; + DWORD wpath_len = info->FileNameLength / sizeof(WCHAR); enum fsmonitor_path_type t; + enum get_relative_result grr; + + if (watch->has_shortnames) { + if (!wcscmp(wpath, watch->dotgit_shortname)) { + /* + * This event exactly matches the + * spelling of the shortname of + * ".git", so we can skip some steps. + * + * (This case is odd because the user + * can "rm -rf GIT~1" and we cannot + * use the filesystem to map it back + * to ".git".) + */ + strbuf_reset(&path); + strbuf_addstr(&path, ".git"); + t = IS_DOT_GIT; + goto process_it; + } - strbuf_reset(&path); - if (normalize_path_in_utf8(info, &path) == -1) - goto skip_this_path; - - t = fsmonitor_classify_path_workdir_relative(path.buf); - - switch (t) { - case IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX: - /* special case cookie files within .git */ - - /* Use just the filename of the cookie file. */ - slash = find_last_dir_sep(path.buf); - string_list_append(&cookie_list, - slash ? slash + 1 : path.buf); - break; - - case IS_INSIDE_DOT_GIT: - /* ignore everything inside of "/.git/" */ - break; + if (watch->has_tilda && !wcschr(wpath, L'~')) { + /* + * Shortnames on this filesystem have tildas + * and the notification path does not have + * one, so we assume that it is a longname. + */ + goto normalize_it; + } - case IS_DOT_GIT: - /* "/.git" was deleted (or renamed away) */ - if ((info->Action == FILE_ACTION_REMOVED) || - (info->Action == FILE_ACTION_RENAMED_OLD_NAME)) { - trace2_data_string("fsmonitor", NULL, - "fsm-listen/dotgit", - "removed"); + grr = get_relative_longname(watch, wpath, wpath_len, + wpath_longname); + switch (grr) { + case GRR_NO_CONVERSION_NEEDED: /* use info buffer as is */ + break; + case GRR_HAVE_CONVERSION: + wpath = wpath_longname; + wpath_len = wcslen(wpath); + break; + default: + case GRR_SHUTDOWN: goto force_shutdown; } - break; + } - case IS_WORKDIR_PATH: - /* queue normal pathname */ - if (!batch) - batch = fsmonitor_batch__new(); - fsmonitor_batch__add_path(batch, path.buf); - break; +normalize_it: + if (normalize_path_in_utf8(wpath, wpath_len, &path) == -1) + goto skip_this_path; - case IS_GITDIR: - case IS_INSIDE_GITDIR: - case IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX: - default: - BUG("unexpected path classification '%d' for '%s'", - t, path.buf); - } + t = fsmonitor_classify_path_workdir_relative(path.buf); + +process_it: + if (process_1_worktree_event(&cookie_list, &batch, &path, t, + info->Action)) + goto force_shutdown; skip_this_path: if (!info->NextEntryOffset) @@ -382,6 +626,9 @@ static int process_worktree_events(struct fsmonitor_daemon_state *state) * Note that we DO NOT get filesystem events on the external * itself (it is not inside something that we are watching). In particular, * we do not get an event if the external is deleted. + * + * Also, we do not care about shortnames within the external , since + * we never send these paths to clients. */ static int process_gitdir_events(struct fsmonitor_daemon_state *state) { @@ -403,8 +650,10 @@ static int process_gitdir_events(struct fsmonitor_daemon_state *state) const char *slash; enum fsmonitor_path_type t; - strbuf_reset(&path); - if (normalize_path_in_utf8(info, &path) == -1) + if (normalize_path_in_utf8( + info->FileName, + info->FileNameLength / sizeof(WCHAR), + &path) == -1) goto skip_this_path; t = fsmonitor_classify_path_gitdir_relative(path.buf); @@ -538,6 +787,8 @@ int fsm_listen__ctor(struct fsmonitor_daemon_state *state) if (!data->watch_worktree) goto failed; + check_for_shortnames(data->watch_worktree); + if (state->nr_paths_watching > 1) { data->watch_gitdir = create_watch(state, state->path_gitdir_watch.buf); diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index a43fc5190f5abe..ee94861fe75d77 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -123,6 +123,71 @@ test_expect_success 'implicit daemon stop (rename .git)' ' test_must_fail git -C test_implicit_2 fsmonitor--daemon status ' +# File systems on Windows may or may not have shortnames. +# This is a volume-specific setting on modern systems. +# "C:/" drives are required to have them enabled. Other +# hard drives default to disabled. +# +# This is a crude test to see if shortnames are enabled +# on the volume containing the test directory. It is +# crude, but it does not require elevation like `fsutil`. +# +test_lazy_prereq SHORTNAMES ' + mkdir .foo && + test -d "FOO~1" +' + +# Here we assume that the shortname of ".git" is "GIT~1". +test_expect_success MINGW,SHORTNAMES 'implicit daemon stop (rename GIT~1)' ' + test_when_finished "stop_daemon_delete_repo test_implicit_1s" && + + git init test_implicit_1s && + + start_daemon test_implicit_1s && + + # renaming the .git directory will implicitly stop the daemon. + # this moves {.git, GIT~1} to {.gitxyz, GITXYZ~1}. + # the rename-from FS Event will contain the shortname. + # + mv test_implicit_1s/GIT~1 test_implicit_1s/.gitxyz && + + sleep 1 && + # put it back so that our status will not crawl out to our + # parent directory. + # this moves {.gitxyz, GITXYZ~1} to {.git, GIT~1}. + mv test_implicit_1s/.gitxyz test_implicit_1s/.git && + + test_must_fail git -C test_implicit_1s fsmonitor--daemon status +' + +# Here we first create a file with LONGNAME of "GIT~1" before +# we create the repo. This will cause the shortname of ".git" +# to be "GIT~2". +test_expect_success MINGW,SHORTNAMES 'implicit daemon stop (rename GIT~2)' ' + test_when_finished "stop_daemon_delete_repo test_implicit_1s2" && + + mkdir test_implicit_1s2 && + echo HELLO >test_implicit_1s2/GIT~1 && + git init test_implicit_1s2 && + + test_path_is_file test_implicit_1s2/GIT~1 && + test_path_is_dir test_implicit_1s2/GIT~2 && + + start_daemon test_implicit_1s2 && + + # renaming the .git directory will implicitly stop the daemon. + # the rename-from FS Event will contain the shortname. + # + mv test_implicit_1s2/GIT~2 test_implicit_1s2/.gitxyz && + + sleep 1 && + # put it back so that our status will not crawl out to our + # parent directory. + mv test_implicit_1s2/.gitxyz test_implicit_1s2/.git && + + test_must_fail git -C test_implicit_1s2 fsmonitor--daemon status +' + test_expect_success 'cannot start multiple daemons' ' test_when_finished "stop_daemon_delete_repo test_multiple" && From b6367c7d0b264c0aa0baf6de864a7d04a03c436f Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 1 Mar 2021 17:48:39 -0500 Subject: [PATCH 085/110] t7527: test status with untracked-cache and fsmonitor--daemon Create 2x2 test matrix with the untracked-cache and fsmonitor--daemon features and a series of edits and verify that status output is identical. Signed-off-by: Jeff Hostetler --- t/t7527-builtin-fsmonitor.sh | 87 ++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index eea9ca1a309ed3..a43fc5190f5abe 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -162,6 +162,8 @@ test_expect_success 'setup' ' .gitignore expect* actual* + flush* + trace* EOF git -c core.useBuiltinFSMonitor= add . && @@ -508,4 +510,89 @@ test_expect_success 'cleanup worktrees' ' stop_daemon_delete_repo wt-base ' +# The next few tests perform arbitrary/contrived file operations and +# confirm that status is correct. That is, that the data (or lack of +# data) from fsmonitor doesn't cause incorrect results. And doesn't +# cause incorrect results when the untracked-cache is enabled. + +test_lazy_prereq UNTRACKED_CACHE ' + { git update-index --test-untracked-cache; ret=$?; } && + test $ret -ne 1 +' + +test_expect_success 'Matrix: setup for untracked-cache,fsmonitor matrix' ' + test_might_fail git config --unset core.useBuiltinFSMonitor && + git update-index --no-fsmonitor && + test_might_fail git fsmonitor--daemon stop +' + +matrix_clean_up_repo () { + git reset --hard HEAD + git clean -fd +} + +matrix_try () { + uc=$1 + fsm=$2 + fn=$3 + + test_expect_success "Matrix[uc:$uc][fsm:$fsm] $fn" ' + matrix_clean_up_repo && + $fn && + if test $uc = false -a $fsm = false + then + git status --porcelain=v1 >.git/expect.$fn + else + git status --porcelain=v1 >.git/actual.$fn + test_cmp .git/expect.$fn .git/actual.$fn + fi + ' + + return $? +} + +uc_values="false" +test_have_prereq UNTRACKED_CACHE && uc_values="false true" +for uc_val in $uc_values +do + if test $uc_val = false + then + test_expect_success "Matrix[uc:$uc_val] disable untracked cache" ' + git config core.untrackedcache false && + git update-index --no-untracked-cache + ' + else + test_expect_success "Matrix[uc:$uc_val] enable untracked cache" ' + git config core.untrackedcache true && + git update-index --untracked-cache + ' + fi + + fsm_values="false true" + for fsm_val in $fsm_values + do + if test $fsm_val = false + then + test_expect_success "Matrix[uc:$uc_val][fsm:$fsm_val] disable fsmonitor" ' + test_might_fail git config --unset core.useBuiltinFSMonitor && + git update-index --no-fsmonitor && + test_might_fail git fsmonitor--daemon stop 2>/dev/null + ' + else + test_expect_success "Matrix[uc:$uc_val][fsm:$fsm_val] enable fsmonitor" ' + git config core.useBuiltinFSMonitor true && + git fsmonitor--daemon start && + git update-index --fsmonitor + ' + fi + + matrix_try $uc_val $fsm_val edit_files + matrix_try $uc_val $fsm_val delete_files + matrix_try $uc_val $fsm_val create_files + matrix_try $uc_val $fsm_val rename_files + matrix_try $uc_val $fsm_val file_to_directory + matrix_try $uc_val $fsm_val directory_to_file + done +done + test_done From ee3be16449fa51255aa09f39582b5f2c694854c5 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 30 Jun 2021 09:30:20 -0400 Subject: [PATCH 086/110] t7527: test FS event reporing on MacOS WRT case and Unicode Confirm that MacOS FS events are reported with a normalized spelling. APFS (and/or HFS+) is case-insensitive. This means that case-independent lookups ( [ -d .git ] and [ -d .GIT ] ) should both succeed. But that doesn't tell us how FS events are reported if we try "rm -rf .git" versus "rm -rf .GIT". Are the events reported using the on-disk spelling of the pathname or in the spelling used by the command. NEEDSWORK: I was only able to test case. It would be nice to add tests that use different Unicode spellings/normalizations and understand the differences between APFS and HFS+ in this area. We should confirm that the spelling of the workdir paths that the daemon sends to clients are always properly normalized. Signed-off-by: Jeff Hostetler --- t/t7527-builtin-fsmonitor.sh | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index ee94861fe75d77..85e249fda42f6d 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -188,6 +188,36 @@ test_expect_success MINGW,SHORTNAMES 'implicit daemon stop (rename GIT~2)' ' test_must_fail git -C test_implicit_1s2 fsmonitor--daemon status ' +# Confirm that MacOS hides all of the Unicode normalization and/or +# case folding from the FS events. That is, are the pathnames in the +# FS events reported using the spelling on the disk or in the spelling +# used by the other process. +# +# Note that we assume that the filesystem is set to case insensitive. +# +# NEEDSWORK: APFS handles Unicode and Unicode normalization +# differently than HFS+. I only have an APFS partition, so +# more testing here would be helpful. +# + +# Rename .git using alternate spelling and confirm that the daemon +# sees the event using the correct spelling and shutdown. +test_expect_success UTF8_NFD_TO_NFC 'MacOS event spelling (rename .GIT)' ' + test_when_finished "stop_daemon_delete_repo test_apfs" && + + git init test_apfs && + start_daemon test_apfs && + + test_path_is_dir test_apfs/.git && + test_path_is_dir test_apfs/.GIT && + + mv test_apfs/.GIT test_apfs/.FOO && + sleep 1 && + mv test_apfs/.FOO test_apfs/.git && + + test_must_fail git -C test_apfs fsmonitor--daemon status +' + test_expect_success 'cannot start multiple daemons' ' test_when_finished "stop_daemon_delete_repo test_multiple" && From a1fc4aad03f3e9f44cce8ef6000863d96fbc02e1 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 12 Jul 2021 12:50:31 -0400 Subject: [PATCH 087/110] t7527: test builtin FSMonitor watching repos with unicode paths Create some test repos with UTF8 pathnames and verify that the builtin FSMonitor can watch them. This test is mainly for Windows where we need to avoid `*A()` routines. Signed-off-by: Jeff Hostetler --- t/t7527-builtin-fsmonitor.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index 85e249fda42f6d..f10a33dde8c270 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -690,4 +690,27 @@ do done done +# Test Unicode UTF-8 characters in the pathname of the working +# directory. Use of "*A()" routines rather than "*W()" routines +# on Windows can sometimes lead to odd failures. +# +u1=$(printf "u_c3_a6__\xC3\xA6") +u2=$(printf "u_e2_99_ab__\xE2\x99\xAB") +u_values="$u1 $u2" +for u in $u_values +do + test_expect_success "Unicode path: $u" ' + test_when_finished "stop_daemon_delete_repo $u" && + + git init "$u" && + echo 1 >"$u"/file1 && + git -C "$u" add file1 && + git -C "$u" config core.useBuiltinFSMonitor true && + + start_daemon "$u" && + git -C "$u" status >actual && + grep "new file: file1" actual + ' +done + test_done From 42d62b4c8828aa7b66eb3d6263f564bfc2c9224b Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 27 Jul 2021 14:06:33 -0400 Subject: [PATCH 088/110] t/helper/fsmonitor-client: create stress test Create a stress test to hammer on the fsmonitor daemon. Create a client-side thread pool of n threads and have each of them make m requests as fast as they can. NEEDSWORK: This is just the client-side thread pool and is useful for interactive testing and experimentation. We need to add a script test to drive this. Signed-off-by: Jeff Hostetler --- t/helper/test-fsmonitor-client.c | 105 +++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/t/helper/test-fsmonitor-client.c b/t/helper/test-fsmonitor-client.c index f7a5b3a32fab86..9dd2f9af553d34 100644 --- a/t/helper/test-fsmonitor-client.c +++ b/t/helper/test-fsmonitor-client.c @@ -7,6 +7,8 @@ #include "cache.h" #include "parse-options.h" #include "fsmonitor-ipc.h" +#include "thread-utils.h" +#include "trace2.h" #ifndef HAVE_FSMONITOR_DAEMON_BACKEND int cmd__fsmonitor_client(int argc, const char **argv) @@ -79,20 +81,120 @@ static int do_send_flush(void) return 0; } +struct hammer_thread_data +{ + pthread_t pthread_id; + int thread_nr; + + int nr_requests; + const char *token; + + int sum_successful; + int sum_errors; +}; + +static void *hammer_thread_proc(void *_hammer_thread_data) +{ + struct hammer_thread_data *data = _hammer_thread_data; + struct strbuf answer = STRBUF_INIT; + int k; + int ret; + + trace2_thread_start("hammer"); + + for (k = 0; k < data->nr_requests; k++) { + strbuf_reset(&answer); + + ret = fsmonitor_ipc__send_query(data->token, &answer); + if (ret < 0) + data->sum_errors++; + else + data->sum_successful++; + } + + strbuf_release(&answer); + trace2_thread_exit(); + return NULL; +} + +/* + * Start a pool of client threads that will each send a series of + * commands to the daemon. + * + * The goal is to overload the daemon with a sustained series of + * concurrent requests. + */ +static int do_hammer(const char *token, int nr_threads, int nr_requests) +{ + struct hammer_thread_data *data = NULL; + int k; + int sum_join_errors = 0; + int sum_commands = 0; + int sum_errors = 0; + + if (!token || !*token) + token = get_token_from_index(); + if (nr_threads < 1) + nr_threads = 1; + if (nr_requests < 1) + nr_requests = 1; + + CALLOC_ARRAY(data, nr_threads); + + for (k = 0; k < nr_threads; k++) { + struct hammer_thread_data *p = &data[k]; + p->thread_nr = k; + p->nr_requests = nr_requests; + p->token = token; + + if (pthread_create(&p->pthread_id, NULL, hammer_thread_proc, p)) { + warning("failed to create thread[%d] skipping remainder", k); + nr_threads = k; + break; + } + } + + for (k = 0; k < nr_threads; k++) { + struct hammer_thread_data *p = &data[k]; + + if (pthread_join(p->pthread_id, NULL)) + sum_join_errors++; + sum_commands += p->sum_successful; + sum_errors += p->sum_errors; + } + + fprintf(stderr, "HAMMER: [threads %d][requests %d] [ok %d][err %d][join %d]\n", + nr_threads, nr_requests, sum_commands, sum_errors, sum_join_errors); + + free(data); + + /* + * TODO Decide if/when to return an error or call die(). + */ + return 0; +} + int cmd__fsmonitor_client(int argc, const char **argv) { const char *subcmd; const char *token = NULL; + int nr_threads = 1; + int nr_requests = 1; const char * const fsmonitor_client_usage[] = { N_("test-helper fsmonitor-client query []"), N_("test-helper fsmonitor-client flush"), + N_("test-helper fsmonitor-client hammer [] [] []"), NULL, }; struct option options[] = { OPT_STRING(0, "token", &token, N_("token"), N_("command token to send to the server")), + + OPT_INTEGER(0, "threads", &nr_threads, N_("number of client threads")), + OPT_INTEGER(0, "requests", &nr_requests, N_("number of requests per thread")), + OPT_END() }; @@ -116,6 +218,9 @@ int cmd__fsmonitor_client(int argc, const char **argv) if (!strcmp(subcmd, "flush")) return !!do_send_flush(); + if (!strcmp(subcmd, "hammer")) + return !!do_hammer(token, nr_threads, nr_requests); + die("Unhandled subcommand: '%s'", subcmd); } #endif From 386c3695b760da3f73204631dbde50582f79325e Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 23 Jun 2021 10:18:17 -0400 Subject: [PATCH 089/110] fsmonitor-settings: bare repos are incompatible with FSMonitor Bare repos do not have a worktree, so there is nothing for the daemon watch. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 11 +++++ builtin/update-index.c | 8 ++++ fsmonitor-settings.c | 83 +++++++++++++++++++++++++++++++++---- fsmonitor-settings.h | 11 +++++ t/t7519-status-fsmonitor.sh | 26 ++++++++++++ 5 files changed, 130 insertions(+), 9 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 6011fe42ee0e5b..899355c55aa8dd 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -1424,6 +1424,17 @@ int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) die(_("invalid 'ipc-threads' value (%d)"), fsmonitor__ipc_threads); + prepare_repo_settings(the_repository); + fsm_settings__set_ipc(the_repository); + + if (fsm_settings__get_mode(the_repository) == FSMONITOR_MODE_INCOMPATIBLE) { + struct strbuf buf_reason = STRBUF_INIT; + fsm_settings__get_reason(the_repository, &buf_reason); + error("%s '%s'", buf_reason.buf, xgetcwd()); + strbuf_release(&buf_reason); + return -1; + } + if (!strcmp(subcmd, "start")) return !!try_to_start_background_daemon(); diff --git a/builtin/update-index.c b/builtin/update-index.c index 25dae7360a601f..38bf0ff4bd5a21 100644 --- a/builtin/update-index.c +++ b/builtin/update-index.c @@ -1218,6 +1218,14 @@ int cmd_update_index(int argc, const char **argv, const char *prefix) if (fsmonitor > 0) { enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); + if (fsm_mode == FSMONITOR_MODE_INCOMPATIBLE) { + struct strbuf buf_reason = STRBUF_INIT; + fsm_settings__get_reason(r, &buf_reason); + error("%s", buf_reason.buf); + strbuf_release(&buf_reason); + return -1; + } + if (fsm_mode == FSMONITOR_MODE_DISABLED) { warning(_("core.useBuiltinFSMonitor is unset; " "set it if you really want to enable the " diff --git a/fsmonitor-settings.c b/fsmonitor-settings.c index 2770266f5ee1ad..5a803a41d57434 100644 --- a/fsmonitor-settings.c +++ b/fsmonitor-settings.c @@ -9,19 +9,57 @@ */ struct fsmonitor_settings { enum fsmonitor_mode mode; + enum fsmonitor_reason reason; char *hook_path; }; -void fsm_settings__set_ipc(struct repository *r) +static void set_incompatible(struct repository *r, + enum fsmonitor_reason reason) { struct fsmonitor_settings *s = r->settings.fsmonitor; + s->mode = FSMONITOR_MODE_INCOMPATIBLE; + s->reason = reason; +} + +static int check_for_incompatible(struct repository *r) +{ + if (!r->worktree) { + /* + * Bare repositories don't have a working directory and + * therefore have nothing to watch. + */ + set_incompatible(r, FSMONITOR_REASON_BARE); + return 1; + } + + return 0; +} + +static struct fsmonitor_settings *s_init(struct repository *r) +{ + if (!r->settings.fsmonitor) + CALLOC_ARRAY(r->settings.fsmonitor, 1); + + return r->settings.fsmonitor; +} + +void fsm_settings__set_ipc(struct repository *r) +{ + struct fsmonitor_settings *s = s_init(r); + + if (check_for_incompatible(r)) + return; + s->mode = FSMONITOR_MODE_IPC; } void fsm_settings__set_hook(struct repository *r, const char *path) { - struct fsmonitor_settings *s = r->settings.fsmonitor; + struct fsmonitor_settings *s = s_init(r); + + if (check_for_incompatible(r)) + return; s->mode = FSMONITOR_MODE_HOOK; s->hook_path = strdup(path); @@ -29,9 +67,10 @@ void fsm_settings__set_hook(struct repository *r, const char *path) void fsm_settings__set_disabled(struct repository *r) { - struct fsmonitor_settings *s = r->settings.fsmonitor; + struct fsmonitor_settings *s = s_init(r); s->mode = FSMONITOR_MODE_DISABLED; + s->reason = FSMONITOR_REASON_ZERO; FREE_AND_NULL(s->hook_path); } @@ -65,12 +104,6 @@ static int check_for_hook(struct repository *r) static void lookup_fsmonitor_settings(struct repository *r) { - struct fsmonitor_settings *s; - - CALLOC_ARRAY(s, 1); - - r->settings.fsmonitor = s; - if (check_for_ipc(r)) return; @@ -95,3 +128,35 @@ const char *fsm_settings__get_hook_path(struct repository *r) return r->settings.fsmonitor->hook_path; } + +static void create_reason_message(struct repository *r, + struct strbuf *buf_reason) +{ + struct fsmonitor_settings *s = r->settings.fsmonitor; + + switch (s->reason) { + case FSMONITOR_REASON_ZERO: + return; + + case FSMONITOR_REASON_BARE: + strbuf_addstr(buf_reason, + _("bare repos are incompatible with fsmonitor")); + return; + + default: + BUG("Unhandled case in create_reason_message '%d'", s->reason); + } +} +enum fsmonitor_reason fsm_settings__get_reason(struct repository *r, + struct strbuf *buf_reason) +{ + strbuf_reset(buf_reason); + + if (!r->settings.fsmonitor) + lookup_fsmonitor_settings(r); + + if (r->settings.fsmonitor->mode == FSMONITOR_MODE_INCOMPATIBLE) + create_reason_message(r, buf_reason); + + return r->settings.fsmonitor->reason; +} diff --git a/fsmonitor-settings.h b/fsmonitor-settings.h index 50b2923461656c..eb45524b1f2297 100644 --- a/fsmonitor-settings.h +++ b/fsmonitor-settings.h @@ -4,17 +4,28 @@ struct repository; enum fsmonitor_mode { + FSMONITOR_MODE_INCOMPATIBLE = -1, /* see _reason */ FSMONITOR_MODE_DISABLED = 0, FSMONITOR_MODE_HOOK = 1, /* core.fsmonitor */ FSMONITOR_MODE_IPC = 2, /* core.useBuiltinFSMonitor */ }; +/* + * Incompatibility reasons. + */ +enum fsmonitor_reason { + FSMONITOR_REASON_ZERO = 0, + FSMONITOR_REASON_BARE = 1, +}; + void fsm_settings__set_ipc(struct repository *r); void fsm_settings__set_hook(struct repository *r, const char *path); void fsm_settings__set_disabled(struct repository *r); enum fsmonitor_mode fsm_settings__get_mode(struct repository *r); const char *fsm_settings__get_hook_path(struct repository *r); +enum fsmonitor_reason fsm_settings__get_reason(struct repository *r, + struct strbuf *buf_reason); struct fsmonitor_settings; diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index deea88d4431d23..b9407d4738b129 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -55,6 +55,32 @@ test_lazy_prereq UNTRACKED_CACHE ' test $ret -ne 1 ' +# Test that we detect and disallow repos that are incompatible with FSMonitor. +test_expect_success 'incompatible bare repo' ' + test_when_finished "rm -rf ./bare-clone actual expect" && + git init --bare bare-clone && + cat >expect <<-\EOF && + error: bare repos are incompatible with fsmonitor + EOF + + test_must_fail \ + git -C ./bare-clone -c core.fsmonitor=foo \ + update-index --fsmonitor 2>actual && + test_cmp expect actual && + + test_must_fail \ + git -C ./bare-clone -c core.usebuiltinfsmonitor=true \ + update-index --fsmonitor 2>actual && + test_cmp expect actual +' + +test_expect_success FSMONITOR_DAEMON 'run fsmonitor-daemon in bare repo' ' + test_when_finished "rm -rf ./bare-clone actual" && + git init --bare bare-clone && + test_must_fail git -C ./bare-clone fsmonitor--daemon run 2>actual && + grep "bare repos are incompatible with fsmonitor" actual +' + test_expect_success 'setup' ' mkdir -p .git/hooks && : >tracked && From b77197af823cbef8ce316e6cfb546f084f4cb785 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 29 Jul 2021 13:33:21 -0400 Subject: [PATCH 090/110] fsmonitor-settings: stub in platform-specific incompatibility checking Extend generic incompatibility checkout with platform-specific mechanism. Stub in Win32 version. In the existing fsmonitor-settings code we have a way to mark types of repos as incompatible with fsmonitor (whether via the hook and ipc APIs). For example, we do this for bare repos, since there are no files to watch. Extend this exclusion mechanism for platfor-specific reasons. This commit just creates the framework and adds a stub for Win32. Signed-off-by: Jeff Hostetler --- Makefile | 13 +++++++++++++ compat/fsmonitor/fsm-settings-win32.c | 9 +++++++++ config.mak.uname | 4 ++++ contrib/buildsystems/CMakeLists.txt | 3 +++ fsmonitor-settings.c | 12 ++++++++++++ fsmonitor-settings.h | 13 +++++++++++++ 6 files changed, 54 insertions(+) create mode 100644 compat/fsmonitor/fsm-settings-win32.c diff --git a/Makefile b/Makefile index dca63dd2d55cc0..89567e4b6612f0 100644 --- a/Makefile +++ b/Makefile @@ -476,6 +476,11 @@ all:: # `compat/fsmonitor/fsm-listen-.c` that implements the # `fsm_listen__*()` routines. # +# If your platform has os-specific ways to tell if a repo is incompatible with +# fsmonitor (whether the hook or ipc daemon version), set FSMONITOR_OS_SETTINGS +# to the "" of the corresponding `compat/fsmonitor/fsm-settings-.c` +# that implements the `fsm_os_settings__*()` routines. +# # Define DEVELOPER to enable more compiler warnings. Compiler version # and family are auto detected, but could be overridden by defining # COMPILER_FEATURES (see config.mak.dev). You can still set @@ -1951,6 +1956,11 @@ ifdef FSMONITOR_DAEMON_BACKEND COMPAT_OBJS += compat/fsmonitor/fsm-listen-$(FSMONITOR_DAEMON_BACKEND).o endif +ifdef FSMONITOR_OS_SETTINGS + COMPAT_CFLAGS += -DHAVE_FSMONITOR_OS_SETTINGS + COMPAT_OBJS += compat/fsmonitor/fsm-settings-$(FSMONITOR_OS_SETTINGS).o +endif + ifeq ($(TCLTK_PATH),) NO_TCLTK = NoThanks endif @@ -2835,6 +2845,9 @@ GIT-BUILD-OPTIONS: FORCE ifdef FSMONITOR_DAEMON_BACKEND @echo FSMONITOR_DAEMON_BACKEND=\''$(subst ','\'',$(subst ','\'',$(FSMONITOR_DAEMON_BACKEND)))'\' >>$@+ endif +ifdef FSMONITOR_OS_SETTINGS + @echo FSMONITOR_OS_SETTINGS=\''$(subst ','\'',$(subst ','\'',$(FSMONITOR_OS_SETTINGS)))'\' >>$@+ +endif ifdef TEST_OUTPUT_DIRECTORY @echo TEST_OUTPUT_DIRECTORY=\''$(subst ','\'',$(subst ','\'',$(TEST_OUTPUT_DIRECTORY)))'\' >>$@+ endif diff --git a/compat/fsmonitor/fsm-settings-win32.c b/compat/fsmonitor/fsm-settings-win32.c new file mode 100644 index 00000000000000..176a6f5726c729 --- /dev/null +++ b/compat/fsmonitor/fsm-settings-win32.c @@ -0,0 +1,9 @@ +#include "cache.h" +#include "config.h" +#include "repository.h" +#include "fsmonitor-settings.h" + +enum fsmonitor_reason fsm_os__incompatible(struct repository *r) +{ + return FSMONITOR_REASON_ZERO; +} diff --git a/config.mak.uname b/config.mak.uname index a15cf715cc1c7d..631771cd845100 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -435,6 +435,8 @@ ifeq ($(uname_S),Windows) # These are always available, so we do not have to conditionally # support it. FSMONITOR_DAEMON_BACKEND = win32 + FSMONITOR_OS_SETTINGS = win32 + NO_SVN_TESTS = YesPlease RUNTIME_PREFIX = YesPlease HAVE_WPGMPTR = YesWeDo @@ -627,6 +629,8 @@ ifneq (,$(findstring MINGW,$(uname_S))) # These are always available, so we do not have to conditionally # support it. FSMONITOR_DAEMON_BACKEND = win32 + FSMONITOR_OS_SETTINGS = win32 + RUNTIME_PREFIX = YesPlease HAVE_WPGMPTR = YesWeDo NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 58bc3383de323e..690db6ec58d5c5 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -296,6 +296,9 @@ if(SUPPORTS_SIMPLE_IPC) if(CMAKE_SYSTEM_NAME STREQUAL "Windows") add_compile_definitions(HAVE_FSMONITOR_DAEMON_BACKEND) list(APPEND compat_SOURCES compat/fsmonitor/fsm-listen-win32.c) + + add_compile_definitions(HAVE_FSMONITOR_OS_SETTINGS) + list(APPEND compat_SOURCES compat/fsmonitor/fsm-settings-win32.c) elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") add_compile_definitions(HAVE_FSMONITOR_DAEMON_BACKEND) list(APPEND compat_SOURCES compat/fsmonitor/fsm-listen-darwin.c) diff --git a/fsmonitor-settings.c b/fsmonitor-settings.c index 5a803a41d57434..408c3f159d6ed6 100644 --- a/fsmonitor-settings.c +++ b/fsmonitor-settings.c @@ -33,6 +33,18 @@ static int check_for_incompatible(struct repository *r) return 1; } +#ifdef HAVE_FSMONITOR_OS_SETTINGS + { + enum fsmonitor_reason reason; + + reason = fsm_os__incompatible(r); + if (reason != FSMONITOR_REASON_ZERO) { + set_incompatible(r, reason); + return 1; + } + } +#endif + return 0; } diff --git a/fsmonitor-settings.h b/fsmonitor-settings.h index eb45524b1f2297..813a5406197c61 100644 --- a/fsmonitor-settings.h +++ b/fsmonitor-settings.h @@ -29,4 +29,17 @@ enum fsmonitor_reason fsm_settings__get_reason(struct repository *r, struct fsmonitor_settings; +#ifdef HAVE_FSMONITOR_OS_SETTINGS +/* + * Ask platform-specific code whether the repository is incompatible + * with fsmonitor (both hook and ipc modes). For example, if the working + * directory is on a remote volume and mounted via a technology that does + * not support notification events. + * + * fsm_os__* routines should considered private to fsm_settings__ + * routines. + */ +enum fsmonitor_reason fsm_os__incompatible(struct repository *r); +#endif /* HAVE_FSMONITOR_OS_SETTINGS */ + #endif /* FSMONITOR_SETTINGS_H */ From d432e98d60b9dac862a09522cb89f97c54aaf094 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 2 Aug 2021 15:40:36 -0400 Subject: [PATCH 091/110] fsmonitor-settings: virtual repos are incompatible with FSMonitor Virtual repos, such as GVFS (aka VFS for Git), are incompatible with FSMonitor. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-settings-win32.c | 26 ++++++++++++++++++++++++++ fsmonitor-settings.c | 5 +++++ fsmonitor-settings.h | 1 + t/t7519-status-fsmonitor.sh | 9 +++++++++ 4 files changed, 41 insertions(+) diff --git a/compat/fsmonitor/fsm-settings-win32.c b/compat/fsmonitor/fsm-settings-win32.c index 176a6f5726c729..7caa79570afa92 100644 --- a/compat/fsmonitor/fsm-settings-win32.c +++ b/compat/fsmonitor/fsm-settings-win32.c @@ -3,7 +3,33 @@ #include "repository.h" #include "fsmonitor-settings.h" +/* + * GVFS (aka VFS for Git) is incompatible with FSMonitor. + * + * Granted, core Git does not know anything about GVFS and we + * shouldn't make assumptions about a downstream feature, but users + * can install both versions. And this can lead to incorrect results + * from core Git commands. So, without bringing in any of the GVFS + * code, do a simple config test for a published config setting. (We + * do not look at the various *_TEST_* environment variables.) + */ +static enum fsmonitor_reason is_virtual(struct repository *r) +{ + const char *const_str; + + if (!repo_config_get_value(r, "core.virtualfilesystem", &const_str)) + return FSMONITOR_REASON_VIRTUAL; + + return FSMONITOR_REASON_ZERO; +} + enum fsmonitor_reason fsm_os__incompatible(struct repository *r) { + enum fsmonitor_reason reason; + + reason = is_virtual(r); + if (reason) + return reason; + return FSMONITOR_REASON_ZERO; } diff --git a/fsmonitor-settings.c b/fsmonitor-settings.c index 408c3f159d6ed6..41cfeea9e85a7a 100644 --- a/fsmonitor-settings.c +++ b/fsmonitor-settings.c @@ -155,6 +155,11 @@ static void create_reason_message(struct repository *r, _("bare repos are incompatible with fsmonitor")); return; + case FSMONITOR_REASON_VIRTUAL: + strbuf_addstr(buf_reason, + _("virtual repos are incompatible with fsmonitor")); + return; + default: BUG("Unhandled case in create_reason_message '%d'", s->reason); } diff --git a/fsmonitor-settings.h b/fsmonitor-settings.h index 813a5406197c61..7c8da4e80a3060 100644 --- a/fsmonitor-settings.h +++ b/fsmonitor-settings.h @@ -16,6 +16,7 @@ enum fsmonitor_mode { enum fsmonitor_reason { FSMONITOR_REASON_ZERO = 0, FSMONITOR_REASON_BARE = 1, + FSMONITOR_REASON_VIRTUAL = 2, }; void fsm_settings__set_ipc(struct repository *r); diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index b9407d4738b129..2867806b5d1194 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -81,6 +81,15 @@ test_expect_success FSMONITOR_DAEMON 'run fsmonitor-daemon in bare repo' ' grep "bare repos are incompatible with fsmonitor" actual ' +test_expect_success MINGW,FSMONITOR_DAEMON 'run fsmonitor-daemon in virtual repo' ' + test_when_finished "rm -rf ./fake-virtual-clone actual" && + git init fake-virtual-clone && + test_must_fail git -C ./fake-virtual-clone \ + -c core.virtualfilesystem=true \ + fsmonitor--daemon run 2>actual && + grep "virtual repos are incompatible with fsmonitor" actual +' + test_expect_success 'setup' ' mkdir -p .git/hooks && : >tracked && From c54c767b1ae9f8027b318aaff06a811cd1cad644 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 4 Aug 2021 10:20:55 -0400 Subject: [PATCH 092/110] fsmonitor-settings: stub in platform-specific incompatibility checking on MacOS Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-settings-darwin.c | 9 +++++++++ config.mak.uname | 1 + contrib/buildsystems/CMakeLists.txt | 3 +++ 3 files changed, 13 insertions(+) create mode 100644 compat/fsmonitor/fsm-settings-darwin.c diff --git a/compat/fsmonitor/fsm-settings-darwin.c b/compat/fsmonitor/fsm-settings-darwin.c new file mode 100644 index 00000000000000..176a6f5726c729 --- /dev/null +++ b/compat/fsmonitor/fsm-settings-darwin.c @@ -0,0 +1,9 @@ +#include "cache.h" +#include "config.h" +#include "repository.h" +#include "fsmonitor-settings.h" + +enum fsmonitor_reason fsm_os__incompatible(struct repository *r) +{ + return FSMONITOR_REASON_ZERO; +} diff --git a/config.mak.uname b/config.mak.uname index 631771cd845100..5deac1b450c2a4 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -153,6 +153,7 @@ ifeq ($(uname_S),Darwin) ifndef NO_PTHREADS ifndef NO_UNIX_SOCKETS FSMONITOR_DAEMON_BACKEND = darwin + FSMONITOR_OS_SETTINGS = darwin endif endif diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 690db6ec58d5c5..e0a5440ca1f1cf 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -302,6 +302,9 @@ if(SUPPORTS_SIMPLE_IPC) elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") add_compile_definitions(HAVE_FSMONITOR_DAEMON_BACKEND) list(APPEND compat_SOURCES compat/fsmonitor/fsm-listen-darwin.c) + + add_compile_definitions(HAVE_FSMONITOR_OS_SETTINGS) + list(APPEND compat_SOURCES compat/fsmonitor/fsm-settings-darwin.c) endif() endif() From 6635d1acfaead7bf8f8cb3ef1e75a423d357bc33 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 4 Aug 2021 16:22:10 -0400 Subject: [PATCH 093/110] fsmonitor-settings: remote repos on MacOS are incompatible with FSMonitor Teach Git to detect remote working directories on MacOS and mark them as incompatible with FSMonitor. With this, `git fsmonitor--daemon run` will error out with a message like it does for bare repos. Client commands, like `git status`, will not attempt to start the daemon. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-settings-darwin.c | 66 ++++++++++++++++++++++++++ fsmonitor-settings.c | 5 ++ fsmonitor-settings.h | 1 + 3 files changed, 72 insertions(+) diff --git a/compat/fsmonitor/fsm-settings-darwin.c b/compat/fsmonitor/fsm-settings-darwin.c index 176a6f5726c729..c3b719d6fb0977 100644 --- a/compat/fsmonitor/fsm-settings-darwin.c +++ b/compat/fsmonitor/fsm-settings-darwin.c @@ -2,8 +2,74 @@ #include "config.h" #include "repository.h" #include "fsmonitor-settings.h" +#include "fsmonitor.h" +#include +#include + +/* + * Remote working directories are problematic for FSMonitor. + * + * The underlying file system on the server machine and/or the remote + * mount type (NFS, SAMBA, etc.) dictates whether notification events + * are available at all to remote client machines. + * + * Kernel differences between the server and client machines also + * dictate the how (buffering, frequency, de-dup) the events are + * delivered to client machine processes. + * + * A client machine (such as a laptop) may choose to suspend/resume + * and it is unclear (without lots of testing) whether the watcher can + * resync after a resume. We might be able to treat this as a normal + * "events were dropped by the kernel" event and do our normal "flush + * and resync" --or-- we might need to close the existing (zombie?) + * notification fd and create a new one. + * + * In theory, the above issues need to be addressed whether we are + * using the Hook or IPC API. + * + * For the builtin FSMonitor, we create the Unix domain socket for the + * IPC in the .git directory. If the working directory is remote, + * then the socket will be created on the remote file system. This + * can fail if the remote file system does not support UDS file types + * (e.g. smbfs to a Windows server) or if the remote kernel does not + * allow a non-local process to bind() the socket. (These problems + * could be fixed by moving the UDS out of the .git directory and to a + * well-known local directory on the client machine, but care should + * be taken to ensure that $HOME is actually local and not a managed + * file share.) + * + * So (for now at least), mark remote working directories as + * incompatible. + */ +static enum fsmonitor_reason is_remote(struct repository *r) +{ + struct statfs fs; + + if (statfs(r->worktree, &fs) == -1) { + int saved_errno = errno; + trace_printf_key(&trace_fsmonitor, "statfs('%s') failed: %s", + r->worktree, strerror(saved_errno)); + errno = saved_errno; + return FSMONITOR_REASON_ZERO; + } + + trace_printf_key(&trace_fsmonitor, + "statfs('%s') [type 0x%08x][flags 0x%08x] '%s'", + r->worktree, fs.f_type, fs.f_flags, fs.f_fstypename); + + if (!(fs.f_flags & MNT_LOCAL)) + return FSMONITOR_REASON_REMOTE; + + return FSMONITOR_REASON_ZERO; +} enum fsmonitor_reason fsm_os__incompatible(struct repository *r) { + enum fsmonitor_reason reason; + + reason = is_remote(r); + if (reason) + return reason; + return FSMONITOR_REASON_ZERO; } diff --git a/fsmonitor-settings.c b/fsmonitor-settings.c index 41cfeea9e85a7a..1aa77b3193e1ff 100644 --- a/fsmonitor-settings.c +++ b/fsmonitor-settings.c @@ -160,6 +160,11 @@ static void create_reason_message(struct repository *r, _("virtual repos are incompatible with fsmonitor")); return; + case FSMONITOR_REASON_REMOTE: + strbuf_addstr(buf_reason, + _("remote repos are incompatible with fsmonitor")); + return; + default: BUG("Unhandled case in create_reason_message '%d'", s->reason); } diff --git a/fsmonitor-settings.h b/fsmonitor-settings.h index 7c8da4e80a3060..da203781725c1f 100644 --- a/fsmonitor-settings.h +++ b/fsmonitor-settings.h @@ -17,6 +17,7 @@ enum fsmonitor_reason { FSMONITOR_REASON_ZERO = 0, FSMONITOR_REASON_BARE = 1, FSMONITOR_REASON_VIRTUAL = 2, + FSMONITOR_REASON_REMOTE = 3, }; void fsm_settings__set_ipc(struct repository *r); From e3c9f93317f5f0dc3978a8077c9beff4343bc700 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 5 Aug 2021 09:07:48 -0400 Subject: [PATCH 094/110] fsmonitor-settings: remote repos on Windows are incompatible with FSMonitor Teach Git to detect remote working directories on Windows and mark them as incompatible with FSMonitor. With this `git fsmonitor--daemon run` will error out with a message like it does for bare repos. Client commands, such as `git status`, will not attempt to start the daemon. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-settings-win32.c | 102 ++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/compat/fsmonitor/fsm-settings-win32.c b/compat/fsmonitor/fsm-settings-win32.c index 7caa79570afa92..77156c0ca20ea4 100644 --- a/compat/fsmonitor/fsm-settings-win32.c +++ b/compat/fsmonitor/fsm-settings-win32.c @@ -2,6 +2,7 @@ #include "config.h" #include "repository.h" #include "fsmonitor-settings.h" +#include "fsmonitor.h" /* * GVFS (aka VFS for Git) is incompatible with FSMonitor. @@ -23,6 +24,103 @@ static enum fsmonitor_reason is_virtual(struct repository *r) return FSMONITOR_REASON_ZERO; } +/* + * Remote working directories are problematic for FSMonitor. + * + * The underlying file system on the server machine and/or the remote + * mount type dictates whether notification events are available at + * all to remote client machines. + * + * Kernel differences between the server and client machines also + * dictate the how (buffering, frequency, de-dup) the events are + * delivered to client machine processes. + * + * A client machine (such as a laptop) may choose to suspend/resume + * and it is unclear (without lots of testing) whether the watcher can + * resync after a resume. We might be able to treat this as a normal + * "events were dropped by the kernel" event and do our normal "flush + * and resync" --or-- we might need to close the existing (zombie?) + * notification fd and create a new one. + * + * In theory, the above issues need to be addressed whether we are + * using the Hook or IPC API. + * + * So (for now at least), mark remote working directories as + * incompatible. + * + * Notes for testing: + * + * (a) Windows allows a network share to be mapped to a drive letter. + * (This is the normal method to access it.) + * + * $ NET USE Z: \\server\share + * $ git -C Z:/repo status + * + * (b) Windows allows a network share to be referenced WITHOUT mapping + * it to drive letter. + * + * $ NET USE \\server\share\dir + * $ git -C //server/share/repo status + * + * (c) Windows allows "SUBST" to create a fake drive mapping to an + * arbitrary path (which may be remote) + * + * $ SUBST Q: Z:\repo + * $ git -C Q:/ status + * + * (d) Windows allows a directory symlink to be created on a local + * file system that points to a remote repo. + * + * $ mklink /d ./link //server/share/repo + * $ git -C ./link status + */ +static enum fsmonitor_reason is_remote(struct repository *r) +{ + wchar_t wpath[MAX_PATH]; + wchar_t wfullpath[MAX_PATH]; + size_t wlen; + UINT driveType; + + /* + * Do everything in wide chars because the drive letter might be + * a multi-byte sequence. See win32_has_dos_drive_prefix(). + */ + if (xutftowcs_path(wpath, r->worktree) < 0) + return FSMONITOR_REASON_ZERO; + + /* + * GetDriveTypeW() requires a final slash. We assume that the + * worktree pathname points to an actual directory. + */ + wlen = wcslen(wpath); + if (wpath[wlen - 1] != L'\\' && wpath[wlen - 1] != L'/') { + wpath[wlen++] = L'\\'; + wpath[wlen] = 0; + } + + /* + * Normalize the path. If nothing else, this converts forward + * slashes to backslashes. This is essential to get GetDriveTypeW() + * correctly handle some UNC "\\server\share\..." paths. + */ + if (!GetFullPathNameW(wpath, MAX_PATH, wfullpath, NULL)) + return FSMONITOR_REASON_ZERO; + + driveType = GetDriveTypeW(wfullpath); + trace_printf_key(&trace_fsmonitor, + "DriveType '%s' L'%ls' (%u)", + r->worktree, wfullpath, driveType); + + if (driveType == DRIVE_REMOTE) { + trace_printf_key(&trace_fsmonitor, + "is_remote('%s') true", + r->worktree); + return FSMONITOR_REASON_REMOTE; + } + + return FSMONITOR_REASON_ZERO; +} + enum fsmonitor_reason fsm_os__incompatible(struct repository *r) { enum fsmonitor_reason reason; @@ -31,5 +129,9 @@ enum fsmonitor_reason fsm_os__incompatible(struct repository *r) if (reason) return reason; + reason = is_remote(r); + if (reason) + return reason; + return FSMONITOR_REASON_ZERO; } From ea6e532d654166f8adcc4bec7d7a6f052cd0aaff Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 23 Aug 2021 17:42:28 -0400 Subject: [PATCH 095/110] unpack-trees: initialize fsmonitor_has_run_once in o->result Initialize `o->result.fsmonitor_has_run_once` based upon value in `o->src_index->fsmonitor_has_run_once` to prevent a second fsmonitor query during the tree traversal and possibly getting a skewed view of the working directory. The checkout code has already talked to the fsmonitor and the traversal is updating the index as it traverses, so there is no need to query the fsmonitor. Signed-off-by: Jeff Hostetler --- unpack-trees.c | 1 + 1 file changed, 1 insertion(+) diff --git a/unpack-trees.c b/unpack-trees.c index e33e7e953eef70..a2c5400bd69215 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -1740,6 +1740,7 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options o->result.fsmonitor_last_update = xstrdup_or_null(o->src_index->fsmonitor_last_update); + o->result.fsmonitor_has_run_once = o->src_index->fsmonitor_has_run_once; /* * Sparse checkout loop #1: set NEW_SKIP_WORKTREE on existing entries From 7059baba90076d0e1ac61cd92371a5a02d0b5c32 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 25 Aug 2021 15:11:11 -0400 Subject: [PATCH 096/110] compat/fsmonitor/fsm-listen-darwin: ignore FSEvents caused by xattr changes on MacOS Ignore FSEvents resulting from `xattr` changes. Git does not care about xattr's or changes to xattr's, so don't waste time collecting these events in the daemon nor transmitting them to clients. Various security tools add xattrs to files and/or directories, such as to mark them as having been downloaded. We should ignore these events since it doesn't affect the content of the file/directory or the normal meta-data that Git cares about. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-listen-darwin.c | 34 +++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/compat/fsmonitor/fsm-listen-darwin.c b/compat/fsmonitor/fsm-listen-darwin.c index 2aefdc14d8917b..79d08517d7b0d3 100644 --- a/compat/fsmonitor/fsm-listen-darwin.c +++ b/compat/fsmonitor/fsm-listen-darwin.c @@ -172,7 +172,7 @@ static void log_flags_set(const char *path, const FSEventStreamEventFlags flag) if (flag & kFSEventStreamEventFlagItemCloned) strbuf_addstr(&msg, "ItemCloned|"); - trace_printf_key(&trace_fsmonitor, "fsevent: '%s', flags=%u %s", + trace_printf_key(&trace_fsmonitor, "fsevent: '%s', flags=0x%x %s", path, flag, msg.buf); strbuf_release(&msg); @@ -197,6 +197,31 @@ static int ef_is_dropped(const FSEventStreamEventFlags ef) ef & kFSEventStreamEventFlagUserDropped); } +/* + * If an `xattr` change is the only reason we received this event, + * then silently ignore it. Git doesn't care about xattr's. We + * have to be careful here because the kernel can combine multiple + * events for a single path. And because events always have certain + * bits set, such as `ItemIsFile` or `ItemIsDir`. + * + * Return 1 if we should ignore it. + */ +static int ef_ignore_xattr(const FSEventStreamEventFlags ef) +{ + static const FSEventStreamEventFlags mask = + kFSEventStreamEventFlagItemChangeOwner | + kFSEventStreamEventFlagItemCreated | + kFSEventStreamEventFlagItemFinderInfoMod | + kFSEventStreamEventFlagItemInodeMetaMod | + kFSEventStreamEventFlagItemModified | + kFSEventStreamEventFlagItemRemoved | + kFSEventStreamEventFlagItemRenamed | + kFSEventStreamEventFlagItemXattrMod | + kFSEventStreamEventFlagItemCloned; + + return ((ef & mask) == kFSEventStreamEventFlagItemXattrMod); +} + static void fsevent_callback(ConstFSEventStreamRef streamRef, void *ctx, size_t num_of_events, @@ -262,6 +287,13 @@ static void fsevent_callback(ConstFSEventStreamRef streamRef, continue; } + if (ef_ignore_xattr(event_flags[k])) { + trace_printf_key(&trace_fsmonitor, + "ignore-xattr: '%s', flags=0x%x", + path_k, event_flags[k]); + continue; + } + switch (fsmonitor_classify_path_absolute(state, path_k)) { case IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX: From e0238f4dfa4fa333d9583c890e74fdf93835e71c Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 30 Aug 2021 13:03:57 -0400 Subject: [PATCH 097/110] fsmonitor--daemon: print start message only if fsmonitor.announceStartup Teach fsmonitor--daemon to print a startup message only when `fsmonitor.announceStartup` is true. This setting is false by default so that the output of client commands, like `git status`, is not changed if the daemon is implicitly started. The message is conditionally printed by "run" and "start" subcommands and is sent to stderr. It contains the path to the work tree root. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 899355c55aa8dd..dd0561cfc51518 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -27,6 +27,9 @@ static int fsmonitor__ipc_threads = 8; #define FSMONITOR__START_TIMEOUT "fsmonitor.starttimeout" static int fsmonitor__start_timeout_sec = 60; +#define FSMONITOR__ANNOUNCE_STARTUP "fsmonitor.announcestartup" +static int fsmonitor__announce_startup = 0; + static int fsmonitor_config(const char *var, const char *value, void *cb) { if (!strcmp(var, FSMONITOR__IPC_THREADS)) { @@ -47,6 +50,16 @@ static int fsmonitor_config(const char *var, const char *value, void *cb) return 0; } + if (!strcmp(var, FSMONITOR__ANNOUNCE_STARTUP)) { + int is_bool; + int i = git_config_bool_or_int(var, value, &is_bool); + if (i < 0) + return error(_("value of '%s' not bool or int: %d"), + var, i); + fsmonitor__announce_startup = i; + return 0; + } + return git_default_config(var, value, cb); } @@ -1307,9 +1320,11 @@ static int try_to_run_foreground_daemon(int free_console) die("fsmonitor--daemon is already running '%s'", the_repository->worktree); - printf(_("running fsmonitor-daemon in '%s'\n"), - the_repository->worktree); - fflush(stdout); + if (fsmonitor__announce_startup) { + fprintf(stderr, _("running fsmonitor-daemon in '%s'\n"), + the_repository->worktree); + fflush(stderr); + } #ifdef GIT_WINDOWS_NATIVE if (free_console) @@ -1360,9 +1375,11 @@ static int try_to_start_background_daemon(void) die("fsmonitor--daemon is already running '%s'", the_repository->worktree); - printf(_("starting fsmonitor-daemon in '%s'\n"), - the_repository->worktree); - fflush(stdout); + if (fsmonitor__announce_startup) { + fprintf(stderr, _("starting fsmonitor-daemon in '%s'\n"), + the_repository->worktree); + fflush(stderr); + } cp.git_cmd = 1; From 6fa4324438c2bc411cdb68df47ded83ddde46cef Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 2 Sep 2021 16:03:40 -0400 Subject: [PATCH 098/110] fsmonitor--daemon: cd out of worktree root Teach the fsmonitor--daemon to CD outside of the worktree before starting up. The common Git startup mechanism causes the CWD of the daemon process to be in the root of the worktree. On Windows, this causes the daemon process to hold a locked handle on the CWD and prevents other processes from moving or deleting the worktree while the daemon is running. CD to HOME before entering main event loops. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 32 +++++++++++++++++++++++++++-- compat/fsmonitor/fsm-listen-win32.c | 22 ++++++++++++++------ fsmonitor--daemon.h | 1 + 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index dd0561cfc51518..e9b3f44d791910 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -1169,11 +1169,11 @@ static int fsmonitor_run_daemon_1(struct fsmonitor_daemon_state *state) * before we need it. */ if (ipc_server_run_async(&state->ipc_server_data, - fsmonitor_ipc__get_path(), &ipc_opts, + state->path_ipc.buf, &ipc_opts, handle_client, state)) return error_errno( _("could not start IPC thread pool on '%s'"), - fsmonitor_ipc__get_path()); + state->path_ipc.buf); /* * Start the fsmonitor listener thread to collect filesystem @@ -1208,6 +1208,7 @@ static int fsmonitor_run_daemon_1(struct fsmonitor_daemon_state *state) static int fsmonitor_run_daemon(void) { struct fsmonitor_daemon_state state; + const char *home; int err; memset(&state, 0, sizeof(state)); @@ -1277,6 +1278,15 @@ static int fsmonitor_run_daemon(void) strbuf_addch(&state.path_cookie_prefix, '/'); + /* + * We create a named-pipe or unix domain socket inside of the + * ".git" directory. (Well, on Windows, we base our named + * pipe in the NPFS on the absolute path of the git + * directory.) + */ + strbuf_init(&state.path_ipc, 0); + strbuf_addstr(&state.path_ipc, absolute_path(fsmonitor_ipc__get_path())); + /* * Confirm that we can create platform-specific resources for the * filesystem listener before we bother starting all the threads. @@ -1286,6 +1296,23 @@ static int fsmonitor_run_daemon(void) goto done; } + /* + * CD out of the worktree root directory. + * + * The common Git startup mechanism causes our CWD to be the + * root of the worktree. On Windows, this causes our process + * to hold a locked handle on the CWD. This prevents the + * worktree from being moved or deleted while the daemon is + * running. + * + * We assume that our FS and IPC listener threads have either + * opened all of the handles that they need or will do + * everything using absolute paths. + */ + home = getenv("HOME"); + if (home && *home && chdir(home)) + die_errno("could not cd home '%s'", home); + err = fsmonitor_run_daemon_1(&state); done: @@ -1298,6 +1325,7 @@ static int fsmonitor_run_daemon(void) strbuf_release(&state.path_worktree_watch); strbuf_release(&state.path_gitdir_watch); strbuf_release(&state.path_cookie_prefix); + strbuf_release(&state.path_ipc); /* * NEEDSWORK: Consider "rm -rf /" diff --git a/compat/fsmonitor/fsm-listen-win32.c b/compat/fsmonitor/fsm-listen-win32.c index eb407b0748cf0f..6985903c95b13e 100644 --- a/compat/fsmonitor/fsm-listen-win32.c +++ b/compat/fsmonitor/fsm-listen-win32.c @@ -398,12 +398,22 @@ static int recv_rdcw_watch(struct one_watch *watch) } /* - * NEEDSWORK: If an external is deleted, the above - * returns an error. I'm not sure that there's anything that - * we can do here other than failing -- the /.git - * link file would be broken anyway. We might try to check - * for that and return a better error message, but I'm not - * sure it is worth it. + * GetOverlappedResult() fails if the watched directory is + * deleted while we were waiting for an overlapped IO to + * complete. The documentation did not list specific errors, + * but I observed ERROR_ACCESS_DENIED (0x05) errors during + * testing. + * + * Note that we only get notificaiton events for events + * *within* the directory, not *on* the directory itself. + * (These might be properies of the parent directory, for + * example). + * + * NEEDSWORK: We might try to check for the deleted directory + * case and return a better error message, but I'm not sure it + * is worth it. + * + * Shutdown if we get any error. */ error("GetOverlappedResult failed on '%s' [GLE %ld]", diff --git a/fsmonitor--daemon.h b/fsmonitor--daemon.h index c16ef0956883da..a777c3a0590e85 100644 --- a/fsmonitor--daemon.h +++ b/fsmonitor--daemon.h @@ -54,6 +54,7 @@ struct fsmonitor_daemon_state { struct fsmonitor_daemon_backend_data *backend_data; struct ipc_server_data *ipc_server_data; + struct strbuf path_ipc; }; /* From e1c65c763d1e0b4e4e0db410c91c8badb2edd697 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 8 Sep 2021 16:11:02 -0400 Subject: [PATCH 099/110] fsmonitor--daemon: prepare for adding health thread Refactor daemon thread startup to make it easier to start a third thread class to monitor the health of the daemon. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index e9b3f44d791910..f42fb2ab6267a7 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -1162,6 +1162,8 @@ static int fsmonitor_run_daemon_1(struct fsmonitor_daemon_state *state) */ .uds_disallow_chdir = 0 }; + int listener_started = 0; + int err = 0; /* * Start the IPC thread pool before the we've started the file @@ -1182,15 +1184,20 @@ static int fsmonitor_run_daemon_1(struct fsmonitor_daemon_state *state) if (pthread_create(&state->listener_thread, NULL, fsm_listen__thread_proc, state) < 0) { ipc_server_stop_async(state->ipc_server_data); - ipc_server_await(state->ipc_server_data); - - return error(_("could not start fsmonitor listener thread")); + err = error(_("could not start fsmonitor listener thread")); + goto cleanup; } + listener_started = 1; /* * The daemon is now fully functional in background threads. + * Our primary thread should now just wait while the threads + * do all the work. + */ +cleanup: + /* * Wait for the IPC thread pool to shutdown (whether by client - * request or from filesystem activity). + * request, from filesystem activity, or an error). */ ipc_server_await(state->ipc_server_data); @@ -1199,10 +1206,16 @@ static int fsmonitor_run_daemon_1(struct fsmonitor_daemon_state *state) * event from the IPC thread pool, but it doesn't hurt to tell * it again. And wait for it to shutdown. */ - fsm_listen__stop_async(state); - pthread_join(state->listener_thread, NULL); + if (listener_started) { + fsm_listen__stop_async(state); + pthread_join(state->listener_thread, NULL); + } - return state->error_code; + if (err) + return err; + if (state->error_code) + return state->error_code; + return 0; } static int fsmonitor_run_daemon(void) From 39f0fde4866569e83b7b70deaa0fe973b8f7587a Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 8 Sep 2021 16:46:42 -0400 Subject: [PATCH 100/110] fsmonitor--daemon: rename listener thread related variables Rename platform-specific listener thread related variables and data types as we prepare to add another backend thread type. [] `struct fsmonitor_daemon_backend_data` becomes `struct fsm_listen_data` [] `state->backend_data` becomes `state->listen_data` [] `state->error_code` becomes `state->listen_error_code` Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 6 +++--- compat/fsmonitor/fsm-listen-darwin.c | 30 ++++++++++++++-------------- compat/fsmonitor/fsm-listen-win32.c | 28 +++++++++++++------------- compat/fsmonitor/fsm-listen.h | 2 +- fsmonitor--daemon.h | 6 +++--- 5 files changed, 36 insertions(+), 36 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index f42fb2ab6267a7..f3fb865a9d83fb 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -1213,8 +1213,8 @@ static int fsmonitor_run_daemon_1(struct fsmonitor_daemon_state *state) if (err) return err; - if (state->error_code) - return state->error_code; + if (state->listen_error_code) + return state->listen_error_code; return 0; } @@ -1229,7 +1229,7 @@ static int fsmonitor_run_daemon(void) hashmap_init(&state.cookies, cookies_cmp, NULL, 0); pthread_mutex_init(&state.main_lock, NULL); pthread_cond_init(&state.cookies_cond, NULL); - state.error_code = 0; + state.listen_error_code = 0; state.current_token_data = fsmonitor_new_token_data(); /* Prepare to (recursively) watch the directory. */ diff --git a/compat/fsmonitor/fsm-listen-darwin.c b/compat/fsmonitor/fsm-listen-darwin.c index 79d08517d7b0d3..87a8476b09ff6e 100644 --- a/compat/fsmonitor/fsm-listen-darwin.c +++ b/compat/fsmonitor/fsm-listen-darwin.c @@ -99,7 +99,7 @@ void FSEventStreamRelease(FSEventStreamRef stream); #include "fsm-listen.h" #include "fsmonitor--daemon.h" -struct fsmonitor_daemon_backend_data +struct fsm_listen_data { CFStringRef cfsr_worktree_path; CFStringRef cfsr_gitdir_path; @@ -230,7 +230,7 @@ static void fsevent_callback(ConstFSEventStreamRef streamRef, const FSEventStreamEventId event_ids[]) { struct fsmonitor_daemon_state *state = ctx; - struct fsmonitor_daemon_backend_data *data = state->backend_data; + struct fsm_listen_data *data = state->listen_data; char **paths = (char **)event_paths; struct fsmonitor_batch *batch = NULL; struct string_list cookie_list = STRING_LIST_INIT_DUP; @@ -419,11 +419,11 @@ int fsm_listen__ctor(struct fsmonitor_daemon_state *state) NULL, NULL }; - struct fsmonitor_daemon_backend_data *data; + struct fsm_listen_data *data; const void *dir_array[2]; CALLOC_ARRAY(data, 1); - state->backend_data = data; + state->listen_data = data; data->cfsr_worktree_path = CFStringCreateWithCString( NULL, state->path_worktree_watch.buf, kCFStringEncodingUTF8); @@ -455,18 +455,18 @@ int fsm_listen__ctor(struct fsmonitor_daemon_state *state) failed: error("Unable to create FSEventStream."); - FREE_AND_NULL(state->backend_data); + FREE_AND_NULL(state->listen_data); return -1; } void fsm_listen__dtor(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data; + struct fsm_listen_data *data; - if (!state || !state->backend_data) + if (!state || !state->listen_data) return; - data = state->backend_data; + data = state->listen_data; if (data->stream) { if (data->stream_started) @@ -476,14 +476,14 @@ void fsm_listen__dtor(struct fsmonitor_daemon_state *state) FSEventStreamRelease(data->stream); } - FREE_AND_NULL(state->backend_data); + FREE_AND_NULL(state->listen_data); } void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data; + struct fsm_listen_data *data; - data = state->backend_data; + data = state->listen_data; data->shutdown_style = SHUTDOWN_EVENT; CFRunLoopStop(data->rl); @@ -491,9 +491,9 @@ void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) void fsm_listen__loop(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data; + struct fsm_listen_data *data; - data = state->backend_data; + data = state->listen_data; data->rl = CFRunLoopGetCurrent(); @@ -510,7 +510,7 @@ void fsm_listen__loop(struct fsmonitor_daemon_state *state) switch (data->shutdown_style) { case FORCE_ERROR_STOP: - state->error_code = -1; + state->listen_error_code = -1; /* fall thru */ case FORCE_SHUTDOWN: ipc_server_stop_async(state->ipc_server_data); @@ -522,7 +522,7 @@ void fsm_listen__loop(struct fsmonitor_daemon_state *state) return; force_error_stop_without_loop: - state->error_code = -1; + state->listen_error_code = -1; ipc_server_stop_async(state->ipc_server_data); return; } diff --git a/compat/fsmonitor/fsm-listen-win32.c b/compat/fsmonitor/fsm-listen-win32.c index 6985903c95b13e..8406f8277dc035 100644 --- a/compat/fsmonitor/fsm-listen-win32.c +++ b/compat/fsmonitor/fsm-listen-win32.c @@ -54,7 +54,7 @@ struct one_watch wchar_t dotgit_shortname[16]; /* for 8.3 name */ }; -struct fsmonitor_daemon_backend_data +struct fsm_listen_data { struct one_watch *watch_worktree; struct one_watch *watch_gitdir; @@ -259,7 +259,7 @@ static enum get_relative_result get_relative_longname( void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) { - SetEvent(state->backend_data->hListener[LISTENER_SHUTDOWN]); + SetEvent(state->listen_data->hListener[LISTENER_SHUTDOWN]); } static struct one_watch *create_watch(struct fsmonitor_daemon_state *state, @@ -333,7 +333,7 @@ static void destroy_watch(struct one_watch *watch) free(watch); } -static int start_rdcw_watch(struct fsmonitor_daemon_backend_data *data, +static int start_rdcw_watch(struct fsm_listen_data *data, struct one_watch *watch) { DWORD dwNotifyFilter = @@ -512,7 +512,7 @@ static int process_1_worktree_event( */ static int process_worktree_events(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data = state->backend_data; + struct fsm_listen_data *data = state->listen_data; struct one_watch *watch = data->watch_worktree; struct strbuf path = STRBUF_INIT; struct string_list cookie_list = STRING_LIST_INIT_DUP; @@ -642,7 +642,7 @@ static int process_worktree_events(struct fsmonitor_daemon_state *state) */ static int process_gitdir_events(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data = state->backend_data; + struct fsm_listen_data *data = state->listen_data; struct one_watch *watch = data->watch_gitdir; struct strbuf path = STRBUF_INIT; struct string_list cookie_list = STRING_LIST_INIT_DUP; @@ -700,11 +700,11 @@ static int process_gitdir_events(struct fsmonitor_daemon_state *state) void fsm_listen__loop(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data = state->backend_data; + struct fsm_listen_data *data = state->listen_data; DWORD dwWait; int result; - state->error_code = 0; + state->listen_error_code = 0; if (start_rdcw_watch(data, data->watch_worktree) == -1) goto force_error_stop; @@ -769,7 +769,7 @@ void fsm_listen__loop(struct fsmonitor_daemon_state *state) } force_error_stop: - state->error_code = -1; + state->listen_error_code = -1; force_shutdown: /* @@ -786,7 +786,7 @@ void fsm_listen__loop(struct fsmonitor_daemon_state *state) int fsm_listen__ctor(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data; + struct fsm_listen_data *data; CALLOC_ARRAY(data, 1); @@ -819,7 +819,7 @@ int fsm_listen__ctor(struct fsmonitor_daemon_state *state) data->nr_listener_handles++; } - state->backend_data = data; + state->listen_data = data; return 0; failed: @@ -832,16 +832,16 @@ int fsm_listen__ctor(struct fsmonitor_daemon_state *state) void fsm_listen__dtor(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data; + struct fsm_listen_data *data; - if (!state || !state->backend_data) + if (!state || !state->listen_data) return; - data = state->backend_data; + data = state->listen_data; CloseHandle(data->hEventShutdown); destroy_watch(data->watch_worktree); destroy_watch(data->watch_gitdir); - FREE_AND_NULL(state->backend_data); + FREE_AND_NULL(state->listen_data); } diff --git a/compat/fsmonitor/fsm-listen.h b/compat/fsmonitor/fsm-listen.h index f0539349baf848..41650bf8972217 100644 --- a/compat/fsmonitor/fsm-listen.h +++ b/compat/fsmonitor/fsm-listen.h @@ -33,7 +33,7 @@ void fsm_listen__dtor(struct fsmonitor_daemon_state *state); * do so if the listener thread receives a normal shutdown signal from * the IPC layer.) * - * It should set `state->error_code` to -1 if the daemon should exit + * It should set `state->listen_error_code` to -1 if the daemon should exit * with an error. */ void fsm_listen__loop(struct fsmonitor_daemon_state *state); diff --git a/fsmonitor--daemon.h b/fsmonitor--daemon.h index a777c3a0590e85..f7de7882517e4e 100644 --- a/fsmonitor--daemon.h +++ b/fsmonitor--daemon.h @@ -33,7 +33,7 @@ void fsmonitor_batch__free_list(struct fsmonitor_batch *batch); */ void fsmonitor_batch__add_path(struct fsmonitor_batch *batch, const char *path); -struct fsmonitor_daemon_backend_data; /* opaque platform-specific data */ +struct fsm_listen_data; /* opaque platform-specific data for listener thread */ struct fsmonitor_daemon_state { pthread_t listener_thread; @@ -50,8 +50,8 @@ struct fsmonitor_daemon_state { int cookie_seq; struct hashmap cookies; - int error_code; - struct fsmonitor_daemon_backend_data *backend_data; + int listen_error_code; + struct fsm_listen_data *listen_data; struct ipc_server_data *ipc_server_data; struct strbuf path_ipc; From 0398d35ff7979b541e5e601674966e4cb5c85183 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 10 Sep 2021 12:11:17 -0400 Subject: [PATCH 101/110] fsmonitor--daemon: stub in health thread Create another thread to watch over the daemon process and automatically shut it down if necessary. This commit creates the basic framework for a "health" thread to monitor the daemon and/or the file system. Later commits will add platform-specific code to do the actual work. The "health" thread is intended to monitor conditions that would be difficult to track inside the IPC thread pool and/or the file system listener threads. For example, when there are file system events outside of the watched worktree root or if we want to have an idle-timeout auto-shutdown feature. Signed-off-by: Jeff Hostetler --- Makefile | 6 ++- builtin/fsmonitor--daemon.c | 39 +++++++++++++++ compat/fsmonitor/fsm-health-darwin.c | 24 ++++++++++ compat/fsmonitor/fsm-health-win32.c | 72 ++++++++++++++++++++++++++++ compat/fsmonitor/fsm-health.h | 47 ++++++++++++++++++ contrib/buildsystems/CMakeLists.txt | 2 + fsmonitor--daemon.h | 4 ++ 7 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 compat/fsmonitor/fsm-health-darwin.c create mode 100644 compat/fsmonitor/fsm-health-win32.c create mode 100644 compat/fsmonitor/fsm-health.h diff --git a/Makefile b/Makefile index 89567e4b6612f0..f0f02bdf83957c 100644 --- a/Makefile +++ b/Makefile @@ -473,8 +473,9 @@ all:: # # If your platform supports a built-in fsmonitor backend, set # FSMONITOR_DAEMON_BACKEND to the "" of the corresponding -# `compat/fsmonitor/fsm-listen-.c` that implements the -# `fsm_listen__*()` routines. +# `compat/fsmonitor/fsm-listen-.c` and +# `compat/fsmonitor/fsm-health-.c` files +# that implement the `fsm_listen__*()` and `fsm_health__*()` routines. # # If your platform has os-specific ways to tell if a repo is incompatible with # fsmonitor (whether the hook or ipc daemon version), set FSMONITOR_OS_SETTINGS @@ -1954,6 +1955,7 @@ endif ifdef FSMONITOR_DAEMON_BACKEND COMPAT_CFLAGS += -DHAVE_FSMONITOR_DAEMON_BACKEND COMPAT_OBJS += compat/fsmonitor/fsm-listen-$(FSMONITOR_DAEMON_BACKEND).o + COMPAT_OBJS += compat/fsmonitor/fsm-health-$(FSMONITOR_DAEMON_BACKEND).o endif ifdef FSMONITOR_OS_SETTINGS diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index f3fb865a9d83fb..591433e897df15 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -3,6 +3,7 @@ #include "parse-options.h" #include "fsmonitor.h" #include "fsmonitor-ipc.h" +#include "compat/fsmonitor/fsm-health.h" #include "compat/fsmonitor/fsm-listen.h" #include "fsmonitor--daemon.h" #include "simple-ipc.h" @@ -1124,6 +1125,18 @@ void fsmonitor_publish(struct fsmonitor_daemon_state *state, pthread_mutex_unlock(&state->main_lock); } +static void *fsm_health__thread_proc(void *_state) +{ + struct fsmonitor_daemon_state *state = _state; + + trace2_thread_start("fsm-health"); + + fsm_health__loop(state); + + trace2_thread_exit(); + return NULL; +} + static void *fsm_listen__thread_proc(void *_state) { struct fsmonitor_daemon_state *state = _state; @@ -1162,6 +1175,7 @@ static int fsmonitor_run_daemon_1(struct fsmonitor_daemon_state *state) */ .uds_disallow_chdir = 0 }; + int health_started = 0; int listener_started = 0; int err = 0; @@ -1189,6 +1203,17 @@ static int fsmonitor_run_daemon_1(struct fsmonitor_daemon_state *state) } listener_started = 1; + /* + * Start the health thread to watch over our process. + */ + if (pthread_create(&state->health_thread, NULL, + fsm_health__thread_proc, state) < 0) { + ipc_server_stop_async(state->ipc_server_data); + err = error(_("could not start fsmonitor health thread")); + goto cleanup; + } + health_started = 1; + /* * The daemon is now fully functional in background threads. * Our primary thread should now just wait while the threads @@ -1211,10 +1236,17 @@ static int fsmonitor_run_daemon_1(struct fsmonitor_daemon_state *state) pthread_join(state->listener_thread, NULL); } + if (health_started) { + fsm_health__stop_async(state); + pthread_join(state->health_thread, NULL); + } + if (err) return err; if (state->listen_error_code) return state->listen_error_code; + if (state->health_error_code) + return state->health_error_code; return 0; } @@ -1230,6 +1262,7 @@ static int fsmonitor_run_daemon(void) pthread_mutex_init(&state.main_lock, NULL); pthread_cond_init(&state.cookies_cond, NULL); state.listen_error_code = 0; + state.health_error_code = 0; state.current_token_data = fsmonitor_new_token_data(); /* Prepare to (recursively) watch the directory. */ @@ -1309,6 +1342,11 @@ static int fsmonitor_run_daemon(void) goto done; } + if (fsm_health__ctor(&state)) { + err = error(_("could not initialize health thread")); + goto done; + } + /* * CD out of the worktree root directory. * @@ -1332,6 +1370,7 @@ static int fsmonitor_run_daemon(void) pthread_cond_destroy(&state.cookies_cond); pthread_mutex_destroy(&state.main_lock); fsm_listen__dtor(&state); + fsm_health__dtor(&state); ipc_server_free(state.ipc_server_data); diff --git a/compat/fsmonitor/fsm-health-darwin.c b/compat/fsmonitor/fsm-health-darwin.c new file mode 100644 index 00000000000000..b9f709e854833f --- /dev/null +++ b/compat/fsmonitor/fsm-health-darwin.c @@ -0,0 +1,24 @@ +#include "cache.h" +#include "config.h" +#include "fsmonitor.h" +#include "fsm-health.h" +#include "fsmonitor--daemon.h" + +int fsm_health__ctor(struct fsmonitor_daemon_state *state) +{ + return 0; +} + +void fsm_health__dtor(struct fsmonitor_daemon_state *state) +{ + return; +} + +void fsm_health__loop(struct fsmonitor_daemon_state *state) +{ + return; +} + +void fsm_health__stop_async(struct fsmonitor_daemon_state *state) +{ +} diff --git a/compat/fsmonitor/fsm-health-win32.c b/compat/fsmonitor/fsm-health-win32.c new file mode 100644 index 00000000000000..94b1d020f25ee2 --- /dev/null +++ b/compat/fsmonitor/fsm-health-win32.c @@ -0,0 +1,72 @@ +#include "cache.h" +#include "config.h" +#include "fsmonitor.h" +#include "fsm-health.h" +#include "fsmonitor--daemon.h" + +struct fsm_health_data +{ + HANDLE hEventShutdown; + + HANDLE hHandles[1]; /* the array does not own these handles */ +#define HEALTH_SHUTDOWN 0 + int nr_handles; /* number of active event handles */ +}; + +int fsm_health__ctor(struct fsmonitor_daemon_state *state) +{ + struct fsm_health_data *data; + + CALLOC_ARRAY(data, 1); + + data->hEventShutdown = CreateEvent(NULL, TRUE, FALSE, NULL); + + data->hHandles[HEALTH_SHUTDOWN] = data->hEventShutdown; + data->nr_handles++; + + state->health_data = data; + return 0; +} + +void fsm_health__dtor(struct fsmonitor_daemon_state *state) +{ + struct fsm_health_data *data; + + if (!state || !state->health_data) + return; + + data = state->health_data; + + CloseHandle(data->hEventShutdown); + + FREE_AND_NULL(state->health_data); +} + +void fsm_health__loop(struct fsmonitor_daemon_state *state) +{ + struct fsm_health_data *data = state->health_data; + + for (;;) { + DWORD dwWait = WaitForMultipleObjects(data->nr_handles, + data->hHandles, + FALSE, INFINITE); + + if (dwWait == WAIT_OBJECT_0 + HEALTH_SHUTDOWN) + goto clean_shutdown; + + error(_("health thread wait failed [GLE %ld]"), + GetLastError()); + goto force_error_stop; + } + +force_error_stop: + state->health_error_code = -1; + ipc_server_stop_async(state->ipc_server_data); +clean_shutdown: + return; +} + +void fsm_health__stop_async(struct fsmonitor_daemon_state *state) +{ + SetEvent(state->health_data->hHandles[HEALTH_SHUTDOWN]); +} diff --git a/compat/fsmonitor/fsm-health.h b/compat/fsmonitor/fsm-health.h new file mode 100644 index 00000000000000..45547ba93804a9 --- /dev/null +++ b/compat/fsmonitor/fsm-health.h @@ -0,0 +1,47 @@ +#ifndef FSM_HEALTH_H +#define FSM_HEALTH_H + +/* This needs to be implemented by each backend */ + +#ifdef HAVE_FSMONITOR_DAEMON_BACKEND + +struct fsmonitor_daemon_state; + +/* + * Initialize platform-specific data for the fsmonitor health thread. + * This will be called from the main thread PRIOR to staring the + * thread. + * + * Returns 0 if successful. + * Returns -1 otherwise. + */ +int fsm_health__ctor(struct fsmonitor_daemon_state *state); + +/* + * Cleanup platform-specific data for the health thread. + * This will be called from the main thread AFTER joining the thread. + */ +void fsm_health__dtor(struct fsmonitor_daemon_state *state); + +/* + * The main body of the platform-specific event loop to monitor the + * health of the daemon process. This will run in the health thread. + * + * The health thread should call `ipc_server_stop_async()` if it needs + * to cause a shutdown. (It should NOT do so if it receives a shutdown + * shutdown signal.) + * + * It should set `state->health_error_code` to -1 if the daemon should exit + * with an error. + */ +void fsm_health__loop(struct fsmonitor_daemon_state *state); + +/* + * Gently request that the health thread shutdown. + * It does not wait for it to stop. The caller should do a JOIN + * to wait for it. + */ +void fsm_health__stop_async(struct fsmonitor_daemon_state *state); + +#endif /* HAVE_FSMONITOR_DAEMON_BACKEND */ +#endif /* FSM_HEALTH_H */ diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index e0a5440ca1f1cf..e7db197f5309cc 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -296,12 +296,14 @@ if(SUPPORTS_SIMPLE_IPC) if(CMAKE_SYSTEM_NAME STREQUAL "Windows") add_compile_definitions(HAVE_FSMONITOR_DAEMON_BACKEND) list(APPEND compat_SOURCES compat/fsmonitor/fsm-listen-win32.c) + list(APPEND compat_SOURCES compat/fsmonitor/fsm-health-win32.c) add_compile_definitions(HAVE_FSMONITOR_OS_SETTINGS) list(APPEND compat_SOURCES compat/fsmonitor/fsm-settings-win32.c) elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") add_compile_definitions(HAVE_FSMONITOR_DAEMON_BACKEND) list(APPEND compat_SOURCES compat/fsmonitor/fsm-listen-darwin.c) + list(APPEND compat_SOURCES compat/fsmonitor/fsm-health-darwin.c) add_compile_definitions(HAVE_FSMONITOR_OS_SETTINGS) list(APPEND compat_SOURCES compat/fsmonitor/fsm-settings-darwin.c) diff --git a/fsmonitor--daemon.h b/fsmonitor--daemon.h index f7de7882517e4e..716e0e4d28dfa5 100644 --- a/fsmonitor--daemon.h +++ b/fsmonitor--daemon.h @@ -34,9 +34,11 @@ void fsmonitor_batch__free_list(struct fsmonitor_batch *batch); void fsmonitor_batch__add_path(struct fsmonitor_batch *batch, const char *path); struct fsm_listen_data; /* opaque platform-specific data for listener thread */ +struct fsm_health_data; /* opaque platform-specific data for health thread */ struct fsmonitor_daemon_state { pthread_t listener_thread; + pthread_t health_thread; pthread_mutex_t main_lock; struct strbuf path_worktree_watch; @@ -51,7 +53,9 @@ struct fsmonitor_daemon_state { struct hashmap cookies; int listen_error_code; + int health_error_code; struct fsm_listen_data *listen_data; + struct fsm_health_data *health_data; struct ipc_server_data *ipc_server_data; struct strbuf path_ipc; From dd4bdde7ded0eb9d5f0be06b1221c5860d6cb1fa Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 10 Sep 2021 12:27:41 -0400 Subject: [PATCH 102/110] compat/fsmonitor/fsm-health-win32: add framework for periodically monitoring Create framework in Win32 version of the "health" thread to periodically inspect the system and shutdown if warranted. This version just include the setup for the timeout in WaitForMultipleObjects() and calls (currently empty) table of functions. A later commit will add functions to the table to actually inspect the system. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-health-win32.c | 54 ++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/compat/fsmonitor/fsm-health-win32.c b/compat/fsmonitor/fsm-health-win32.c index 94b1d020f25ee2..3c3453369cd82d 100644 --- a/compat/fsmonitor/fsm-health-win32.c +++ b/compat/fsmonitor/fsm-health-win32.c @@ -4,6 +4,40 @@ #include "fsm-health.h" #include "fsmonitor--daemon.h" +/* + * Every minute wake up and test our health. + */ +#define WAIT_FREQ_MS (60 * 1000) + +enum interval_fn_ctx { CTX_INIT = 0, CTX_TERM, CTX_TIMER }; + +typedef int (interval_fn)(struct fsmonitor_daemon_state *state, + enum interval_fn_ctx ctx); + +static interval_fn *table[] = { + NULL, /* must be last */ +}; + +/* + * Call all of the functions in the table. + * Shortcut and return first error. + * + * Return 0 if all succeeded. + */ +static int call_all(struct fsmonitor_daemon_state *state, + enum interval_fn_ctx ctx) +{ + int k; + + for (k = 0; table[k]; k++) { + int r = table[k](state, ctx); + if (r) + return r; + } + + return 0; +} + struct fsm_health_data { HANDLE hEventShutdown; @@ -45,15 +79,31 @@ void fsm_health__dtor(struct fsmonitor_daemon_state *state) void fsm_health__loop(struct fsmonitor_daemon_state *state) { struct fsm_health_data *data = state->health_data; + int r; + + r = call_all(state, CTX_INIT); + if (r < 0) + goto force_error_stop; + if (r > 0) + goto force_shutdown; for (;;) { DWORD dwWait = WaitForMultipleObjects(data->nr_handles, data->hHandles, - FALSE, INFINITE); + FALSE, WAIT_FREQ_MS); if (dwWait == WAIT_OBJECT_0 + HEALTH_SHUTDOWN) goto clean_shutdown; + if (dwWait == WAIT_TIMEOUT) { + r = call_all(state, CTX_TIMER); + if (r < 0) + goto force_error_stop; + if (r > 0) + goto force_shutdown; + continue; + } + error(_("health thread wait failed [GLE %ld]"), GetLastError()); goto force_error_stop; @@ -61,8 +111,10 @@ void fsm_health__loop(struct fsmonitor_daemon_state *state) force_error_stop: state->health_error_code = -1; +force_shutdown: ipc_server_stop_async(state->ipc_server_data); clean_shutdown: + call_all(state, CTX_TERM); return; } From 941ff16ce6b0cac00ae4668dd3de139a66afc5aa Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 10 Sep 2021 13:35:37 -0400 Subject: [PATCH 103/110] compat/fsmonitor/fsm-health-win32: force shutdown daemon if worktree root moves Force shutdown fsmonitor daemon if the worktree root directory is moved, renamed, or deleted. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-health-win32.c | 133 ++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/compat/fsmonitor/fsm-health-win32.c b/compat/fsmonitor/fsm-health-win32.c index 3c3453369cd82d..2526ad9194f626 100644 --- a/compat/fsmonitor/fsm-health-win32.c +++ b/compat/fsmonitor/fsm-health-win32.c @@ -14,7 +14,10 @@ enum interval_fn_ctx { CTX_INIT = 0, CTX_TERM, CTX_TIMER }; typedef int (interval_fn)(struct fsmonitor_daemon_state *state, enum interval_fn_ctx ctx); +static interval_fn has_worktree_moved; + static interval_fn *table[] = { + has_worktree_moved, NULL, /* must be last */ }; @@ -45,6 +48,12 @@ struct fsm_health_data HANDLE hHandles[1]; /* the array does not own these handles */ #define HEALTH_SHUTDOWN 0 int nr_handles; /* number of active event handles */ + + struct wt_moved + { + wchar_t wpath[MAX_PATH + 1]; + BY_HANDLE_FILE_INFORMATION bhfi; + } wt_moved; }; int fsm_health__ctor(struct fsmonitor_daemon_state *state) @@ -76,6 +85,130 @@ void fsm_health__dtor(struct fsmonitor_daemon_state *state) FREE_AND_NULL(state->health_data); } +static int lookup_bhfi(wchar_t *wpath, + BY_HANDLE_FILE_INFORMATION *bhfi) +{ + DWORD desired_access = FILE_LIST_DIRECTORY; + DWORD share_mode = + FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE; + HANDLE hDir; + + hDir = CreateFileW(wpath, desired_access, share_mode, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hDir == INVALID_HANDLE_VALUE) { + error(_("[GLE %ld] health thread could not open '%ls'"), + GetLastError(), wpath); + return -1; + } + + if (!GetFileInformationByHandle(hDir, bhfi)) { + error(_("[GLE %ld] health thread getting BHFI for '%ls'"), + GetLastError(), wpath); + CloseHandle(hDir); + return -1; + } + + CloseHandle(hDir); + return 0; +} + +static int bhfi_eq(const BY_HANDLE_FILE_INFORMATION *bhfi_1, + const BY_HANDLE_FILE_INFORMATION *bhfi_2) +{ + return (bhfi_1->dwVolumeSerialNumber == bhfi_2->dwVolumeSerialNumber && + bhfi_1->nFileIndexHigh == bhfi_2->nFileIndexHigh && + bhfi_1->nFileIndexLow == bhfi_2->nFileIndexLow); +} + +/* + * Shutdown if the original worktree root directory been deleted, + * moved, or renamed? + * + * Since the main thread did a "chdir(getenv($HOME))" and our CWD + * is not in the worktree root directory and because the listener + * thread added FILE_SHARE_DELETE to the watch handle, it is possible + * for the root directory to be moved or deleted while we are still + * watching it. We want to detect that here and force a shutdown. + * + * Granted, a delete MAY cause some operations to fail, such as + * GetOverlappedResult(), but it is not guaranteed. And because + * ReadDirectoryChangesW() only reports on changes *WITHIN* the + * directory, not changes *ON* the directory, our watch will not + * receive a delete event for it. + * + * A move/rename of the worktree root will also not generate an event. + * And since the listener thread already has an open handle, it may + * continue to receive events for events within the directory. + * However, the pathname of the named-pipe was constructed using the + * original location of the worktree root. (Remember named-pipes are + * stored in the NPFS and not in the actual file system.) Clients + * trying to talk to the worktree after the move/rename will not + * reach our daemon process, since we're still listening on the + * pipe with original path. + * + * Furthermore, if the user does something like: + * + * $ mv repo repo.old + * $ git init repo + * + * A new daemon cannot be started in the new instance of "repo" + * because the named-pipe is still being used by the daemon on + * the original instance. + * + * So, detect move/rename/delete and shutdown. This should also + * handle unsafe drive removal. + * + * We use the file system unique ID to distinguish the original + * directory instance from a new instance and force a shutdown + * if the unique ID changes. + * + * Since a worktree move/rename/delete/unmount doesn't happen + * that often (and we can't get an immediate event anyway), we + * use a timeout and periodically poll it. + */ +static int has_worktree_moved(struct fsmonitor_daemon_state *state, + enum interval_fn_ctx ctx) +{ + struct fsm_health_data *data = state->health_data; + BY_HANDLE_FILE_INFORMATION bhfi; + int r; + + switch (ctx) { + case CTX_TERM: + return 0; + + case CTX_INIT: + if (xutftowcs_path(data->wt_moved.wpath, + state->path_worktree_watch.buf) < 0) { + error(_("could not convert to wide characters: '%s'"), + state->path_worktree_watch.buf); + return -1; + } + + /* + * On the first call we lookup the unique sequence ID for + * the worktree root directory. + */ + return lookup_bhfi(data->wt_moved.wpath, &data->wt_moved.bhfi); + + case CTX_TIMER: + r = lookup_bhfi(data->wt_moved.wpath, &bhfi); + if (r) + return r; + if (!bhfi_eq(&data->wt_moved.bhfi, &bhfi)) { + error(_("BHFI changed '%ls'"), data->wt_moved.wpath); + return -1; + } + return 0; + + default: + die("unhandled case in 'has_worktree_moved': %d", + (int)ctx); + } + + return 0; +} + void fsm_health__loop(struct fsmonitor_daemon_state *state) { struct fsm_health_data *data = state->health_data; From 991e668afb9347131ce33d2832eed4a097838cab Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 10 Sep 2021 14:59:48 -0400 Subject: [PATCH 104/110] compat/fsmonitor/fsm-listen-darwin: shutdown daemon if worktree root is moved/renamed Teach the listener thread to shutdown the daemon if the spelling of the worktree root directory changes. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-listen-darwin.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/compat/fsmonitor/fsm-listen-darwin.c b/compat/fsmonitor/fsm-listen-darwin.c index 87a8476b09ff6e..d3afbbc53d68bb 100644 --- a/compat/fsmonitor/fsm-listen-darwin.c +++ b/compat/fsmonitor/fsm-listen-darwin.c @@ -178,6 +178,11 @@ static void log_flags_set(const char *path, const FSEventStreamEventFlags flag) strbuf_release(&msg); } +static int ef_is_root_changed(const FSEventStreamEventFlags ef) +{ + return (ef & kFSEventStreamEventFlagRootChanged); +} + static int ef_is_root_delete(const FSEventStreamEventFlags ef) { return (ef & kFSEventStreamEventFlagItemIsDir && @@ -287,6 +292,26 @@ static void fsevent_callback(ConstFSEventStreamRef streamRef, continue; } + if (ef_is_root_changed(event_flags[k])) { + /* + * The spelling of the pathname of the root directory + * has changed. This includes the name of the root + * directory itself of of any parent directory in the + * path. + * + * (There may be other conditions that throw this, + * but I couldn't find any information on it.) + * + * Force a shutdown now and avoid things getting + * out of sync. The Unix domain socket is inside + * the .git directory and a spelling change will make + * it hard for clients to rendezvous with us. + */ + trace_printf_key(&trace_fsmonitor, + "event: root changed"); + goto force_shutdown; + } + if (ef_ignore_xattr(event_flags[k])) { trace_printf_key(&trace_fsmonitor, "ignore-xattr: '%s', flags=0x%x", From edb45f063b64e5b29648e586acd8a72ccfd93d18 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 21 Sep 2021 10:49:19 -0400 Subject: [PATCH 105/110] fsmonitor: measure time taken to apply fsmonitor query result Signed-off-by: Jeff Hostetler --- fsmonitor.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fsmonitor.c b/fsmonitor.c index 3a63cda58f73f0..2befde1ffd2d34 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -403,6 +403,8 @@ void refresh_fsmonitor(struct index_state *istate) * information and that we should consider everything * invalid. We call this a trivial response. */ + trace2_region_enter("fsmonitor", "apply_results", istate->repo); + if (query_success && query_result.buf[bol] != '/') { /* * Mark all pathnames returned by the monitor as dirty. @@ -431,6 +433,9 @@ void refresh_fsmonitor(struct index_state *istate) if (count > fsmonitor_force_update_threshold) istate->cache_changed |= FSMONITOR_CHANGED; + trace2_data_intmax("fsmonitor", istate->repo, "apply_count", + count); + } else { /* * We received a trivial response, so invalidate everything. @@ -458,6 +463,8 @@ void refresh_fsmonitor(struct index_state *istate) if (istate->untracked) istate->untracked->use_fsmonitor = 0; } + trace2_region_leave("fsmonitor", "apply_results", istate->repo); + strbuf_release(&query_result); /* Now that we've updated istate, save the last_update_token */ From 13f8336b2b0603d31320fbb805175dc457db2b74 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 21 Sep 2021 15:06:33 -0400 Subject: [PATCH 106/110] fsmonitor: optimize processing of directory events Teach Git to perform binary search over the cache-entries for a directory notification and then linearly scan forward to find the immediate children. Previously, when the FSMonitor reported a modified directory Git would perform a linear search on the entire cache-entry array for all entries matching that directory prefix and invalidate them. Since the cache-entry array is already sorted, we can use a binary search to find the first matching entry and then only linearly walk forward and invalidate entries until the prefix changes. Also, the original code would invalidate anything having the same directory prefix. Since a directory event should only be received for items that are immediately within the directory (and not within sub-directories of it), only invalidate those entries and not the whole subtree. Signed-off-by: Jeff Hostetler --- fsmonitor.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/fsmonitor.c b/fsmonitor.c index 2befde1ffd2d34..b973e30aa98485 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -199,25 +199,24 @@ static void fsmonitor_refresh_callback(struct index_state *istate, char *name) { int i, len = strlen(name); if (name[len - 1] == '/') { - - /* - * TODO We should binary search to find the first path with - * TODO this directory prefix. Then linearly update entries - * TODO while the prefix matches. Taking care to search without - * TODO the trailing slash -- because '/' sorts after a few - * TODO interesting special chars, like '.' and ' '. - */ + const char *rel; + int pos = index_name_pos(istate, name, len); + if (pos < 0) + pos = -pos - 1; /* Mark all entries for the folder invalid */ - for (i = 0; i < istate->cache_nr; i++) { - if (istate->cache[i]->ce_flags & CE_FSMONITOR_VALID && - starts_with(istate->cache[i]->name, name)) + for (i = pos; i < istate->cache_nr; i++) { + if (!starts_with(istate->cache[i]->name, name)) + break; + /* Only mark the immediate children in the folder */ + rel = istate->cache[i]->name + len; + if (!strchr(rel, '/')) istate->cache[i]->ce_flags &= ~CE_FSMONITOR_VALID; } /* Need to remove the / from the path for the untracked cache */ name[len - 1] = '\0'; } else { - int pos = index_name_pos(istate, name, strlen(name)); + int pos = index_name_pos(istate, name, len); if (pos >= 0) { struct cache_entry *ce = istate->cache[pos]; From 9c05fd9820d4bf34ce73ed77ee76a76f1f67ddd6 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Mar 2021 23:07:26 +0100 Subject: [PATCH 107/110] fsmonitor: mark the built-in FSMonitor as experimental Signed-off-by: Johannes Schindelin Signed-off-by: Jeff Hostetler --- Documentation/config/core.txt | 4 ++-- Documentation/git-fsmonitor--daemon.txt | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 323f966a09e6fd..2db16334252271 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -71,7 +71,7 @@ git by avoiding unnecessary scanning of files that have not changed. + See the "fsmonitor-watchman" section of linkgit:githooks[5]. + -Note: The value of this config setting is ignored if the +Note: The value of this config setting is ignored if the (experimental) built-in file system monitor is enabled (see `core.useBuiltinFSMonitor`). core.fsmonitorHookVersion:: @@ -91,7 +91,7 @@ Note: The value of this config setting is ignored if the built-in file system monitor is enabled (see `core.useBuiltinFSMonitor`). core.useBuiltinFSMonitor:: - If set to true, enable the built-in file system monitor + (EXPERIMENTAL) If set to true, enable the built-in file system monitor daemon for this working directory (linkgit:git-fsmonitor--daemon[1]). + Like hook-based file system monitors, the built-in file system monitor diff --git a/Documentation/git-fsmonitor--daemon.txt b/Documentation/git-fsmonitor--daemon.txt index 154e7684daae07..cf1d4556a51425 100644 --- a/Documentation/git-fsmonitor--daemon.txt +++ b/Documentation/git-fsmonitor--daemon.txt @@ -3,7 +3,7 @@ git-fsmonitor--daemon(1) NAME ---- -git-fsmonitor--daemon - A Built-in File System Monitor +git-fsmonitor--daemon - (EXPERIMENTAL) A Built-in File System Monitor SYNOPSIS -------- @@ -16,6 +16,9 @@ SYNOPSIS DESCRIPTION ----------- +NOTE! This command is still only an experiment, subject to change dramatically +(or even to be abandoned). + A daemon to watch the working directory for file and directory changes using platform-specific file system notification facilities. From 1e54c4a08dcbab156fb88914e9ee99560413ff34 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Mar 2021 23:12:11 +0100 Subject: [PATCH 108/110] Enable the built-in FSMonitor as an experimental feature If `feature.experimental` and `feature.manyFiles` are set and the user has not explicitly turned off the builtin FSMonitor, we now start the built-in FSMonitor by default. Only forcing it when UNSET matches the behavior of UPDATE_DEFAULT_BOOL() used for other repo settings. Signed-off-by: Johannes Schindelin Signed-off-by: Jeff Hostetler --- repo-settings.c | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/repo-settings.c b/repo-settings.c index 08c60a927473e2..5241a9b4a06fba 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -2,13 +2,14 @@ #include "config.h" #include "repository.h" #include "midx.h" -#include "compat/fsmonitor/fsm-listen.h" +#include "fsmonitor-ipc.h" +#include "fsmonitor-settings.h" #define UPDATE_DEFAULT_BOOL(s,v) do { if (s == -1) { s = v; } } while(0) void prepare_repo_settings(struct repository *r) { - int value; + int value, feature_many_files = 0; char *strval; if (r->settings.initialized) @@ -62,6 +63,7 @@ void prepare_repo_settings(struct repository *r) UPDATE_DEFAULT_BOOL(r->settings.core_multi_pack_index, 1); if (!repo_config_get_bool(r, "feature.manyfiles", &value) && value) { + feature_many_files = 1; UPDATE_DEFAULT_BOOL(r->settings.index_version, 4); UPDATE_DEFAULT_BOOL(r->settings.core_untracked_cache, UNTRACKED_CACHE_WRITE); } @@ -70,9 +72,33 @@ void prepare_repo_settings(struct repository *r) r->settings.fetch_write_commit_graph = value; UPDATE_DEFAULT_BOOL(r->settings.fetch_write_commit_graph, 0); - if (!repo_config_get_bool(r, "feature.experimental", &value) && value) + if (!repo_config_get_bool(r, "feature.experimental", &value) && value) { UPDATE_DEFAULT_BOOL(r->settings.fetch_negotiation_algorithm, FETCH_NEGOTIATION_SKIPPING); + /* + * Force enable the builtin FSMonitor (unless the repo + * is incompatible or they've already selected it or + * the hook version). But only if they haven't + * explicitly turned it off -- so only if our config + * value is UNSET. + * + * lookup_fsmonitor_settings() and check_for_ipc() do + * not distinguish between explicitly set FALSE and + * UNSET, so we re-test for an UNSET config key here. + * + * I'm not sure I want to fix fsmonitor-settings.c to + * have more than one _DISABLED state since our usage + * here is only to support this experimental period + * (and I don't want to overload the _reason field + * because it describes incompabilities). + */ + if (feature_many_files && + fsmonitor_ipc__is_supported() && + fsm_settings__get_mode(r) == FSMONITOR_MODE_DISABLED && + repo_config_get_bool(r, "core.usebuiltinfsmonitor", &value)) + fsm_settings__set_ipc(r); + } + /* Hack for test programs like test-dump-untracked-cache */ if (ignore_untracked_cache_config) r->settings.core_untracked_cache = UNTRACKED_CACHE_KEEP; From f715125f28f92031640c3c766bdd0c96c39be91a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 5 Aug 2021 21:28:15 +0200 Subject: [PATCH 109/110] compat/fsmonitor/fsm-*-win32: support long paths Update wchar_t buffers to use MAX_LONG_PATH instead of MAX_PATH in the Win32 backend source files. Signed-off-by: Johannes Schindelin Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-health-win32.c | 6 +++--- compat/fsmonitor/fsm-listen-win32.c | 24 ++++++++++++------------ compat/fsmonitor/fsm-settings-win32.c | 8 ++++---- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/compat/fsmonitor/fsm-health-win32.c b/compat/fsmonitor/fsm-health-win32.c index 2526ad9194f626..2c3d7e4b2ae79a 100644 --- a/compat/fsmonitor/fsm-health-win32.c +++ b/compat/fsmonitor/fsm-health-win32.c @@ -51,7 +51,7 @@ struct fsm_health_data struct wt_moved { - wchar_t wpath[MAX_PATH + 1]; + wchar_t wpath[MAX_LONG_PATH + 1]; BY_HANDLE_FILE_INFORMATION bhfi; } wt_moved; }; @@ -178,8 +178,8 @@ static int has_worktree_moved(struct fsmonitor_daemon_state *state, return 0; case CTX_INIT: - if (xutftowcs_path(data->wt_moved.wpath, - state->path_worktree_watch.buf) < 0) { + if (xutftowcs_long_path(data->wt_moved.wpath, + state->path_worktree_watch.buf) < 0) { error(_("could not convert to wide characters: '%s'"), state->path_worktree_watch.buf); return -1; diff --git a/compat/fsmonitor/fsm-listen-win32.c b/compat/fsmonitor/fsm-listen-win32.c index 8406f8277dc035..eb619d3b980b66 100644 --- a/compat/fsmonitor/fsm-listen-win32.c +++ b/compat/fsmonitor/fsm-listen-win32.c @@ -25,7 +25,7 @@ struct one_watch DWORD count; struct strbuf path; - wchar_t wpath_longname[MAX_PATH + 1]; + wchar_t wpath_longname[MAX_LONG_PATH + 1]; DWORD wpath_longname_len; HANDLE hDir; @@ -128,8 +128,8 @@ static int normalize_path_in_utf8(wchar_t *wpath, DWORD wpath_len, */ static void check_for_shortnames(struct one_watch *watch) { - wchar_t buf_in[MAX_PATH + 1]; - wchar_t buf_out[MAX_PATH + 1]; + wchar_t buf_in[MAX_LONG_PATH + 1]; + wchar_t buf_out[MAX_LONG_PATH + 1]; wchar_t *last_slash = NULL; wchar_t *last_bslash = NULL; wchar_t *last; @@ -138,7 +138,7 @@ static void check_for_shortnames(struct one_watch *watch) wcscpy(buf_in, watch->wpath_longname); wcscpy(buf_in + watch->wpath_longname_len, L".git"); - if (!GetShortPathNameW(buf_in, buf_out, MAX_PATH)) + if (!GetShortPathNameW(buf_in, buf_out, MAX_LONG_PATH)) return; last_slash = wcsrchr(buf_out, L'/'); @@ -196,8 +196,8 @@ static enum get_relative_result get_relative_longname( const wchar_t *wpath, DWORD wpath_len, wchar_t *wpath_longname) { - wchar_t buf_in[2 * MAX_PATH + 1]; - wchar_t buf_out[MAX_PATH + 1]; + wchar_t buf_in[2 * MAX_LONG_PATH + 1]; + wchar_t buf_out[MAX_LONG_PATH + 1]; DWORD root_len; /* Build L"/" */ @@ -211,7 +211,7 @@ static enum get_relative_result get_relative_longname( * shortname or a longname. This routine allows either to be * given as input. */ - if (!GetLongPathNameW(buf_in, buf_out, MAX_PATH)) { + if (!GetLongPathNameW(buf_in, buf_out, MAX_LONG_PATH)) { /* * The shortname to longname conversion can fail for * various reasons, for example if the file has been @@ -271,10 +271,10 @@ static struct one_watch *create_watch(struct fsmonitor_daemon_state *state, FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE; HANDLE hDir; DWORD len_longname; - wchar_t wpath[MAX_PATH + 1]; - wchar_t wpath_longname[MAX_PATH + 1]; + wchar_t wpath[MAX_LONG_PATH + 1]; + wchar_t wpath_longname[MAX_LONG_PATH + 1]; - if (xutftowcs_path(wpath, path) < 0) { + if (xutftowcs_long_path(wpath, path) < 0) { error(_("could not convert to wide characters: '%s'"), path); return NULL; } @@ -289,7 +289,7 @@ static struct one_watch *create_watch(struct fsmonitor_daemon_state *state, return NULL; } - if (!GetLongPathNameW(wpath, wpath_longname, MAX_PATH)) { + if (!GetLongPathNameW(wpath, wpath_longname, MAX_LONG_PATH)) { error(_("[GLE %ld] could not get longname of '%s'"), GetLastError(), path); CloseHandle(hDir); @@ -518,7 +518,7 @@ static int process_worktree_events(struct fsmonitor_daemon_state *state) struct string_list cookie_list = STRING_LIST_INIT_DUP; struct fsmonitor_batch *batch = NULL; const char *p = watch->buffer; - wchar_t wpath_longname[MAX_PATH + 1]; + wchar_t wpath_longname[MAX_LONG_PATH + 1]; /* * If the kernel gets more events than will fit in the kernel diff --git a/compat/fsmonitor/fsm-settings-win32.c b/compat/fsmonitor/fsm-settings-win32.c index 77156c0ca20ea4..d775c082451d8a 100644 --- a/compat/fsmonitor/fsm-settings-win32.c +++ b/compat/fsmonitor/fsm-settings-win32.c @@ -76,8 +76,8 @@ static enum fsmonitor_reason is_virtual(struct repository *r) */ static enum fsmonitor_reason is_remote(struct repository *r) { - wchar_t wpath[MAX_PATH]; - wchar_t wfullpath[MAX_PATH]; + wchar_t wpath[MAX_LONG_PATH]; + wchar_t wfullpath[MAX_LONG_PATH]; size_t wlen; UINT driveType; @@ -85,7 +85,7 @@ static enum fsmonitor_reason is_remote(struct repository *r) * Do everything in wide chars because the drive letter might be * a multi-byte sequence. See win32_has_dos_drive_prefix(). */ - if (xutftowcs_path(wpath, r->worktree) < 0) + if (xutftowcs_long_path(wpath, r->worktree) < 0) return FSMONITOR_REASON_ZERO; /* @@ -103,7 +103,7 @@ static enum fsmonitor_reason is_remote(struct repository *r) * slashes to backslashes. This is essential to get GetDriveTypeW() * correctly handle some UNC "\\server\share\..." paths. */ - if (!GetFullPathNameW(wpath, MAX_PATH, wfullpath, NULL)) + if (!GetFullPathNameW(wpath, MAX_LONG_PATH, wfullpath, NULL)) return FSMONITOR_REASON_ZERO; driveType = GetDriveTypeW(wfullpath); From e4dcae60785e2f4b2f443ae0a48719b566799408 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 5 Oct 2021 10:35:09 -0400 Subject: [PATCH 110/110] fixup! gvfs: disable the built-in FSMonitor Signed-off-by: Jeff Hostetler --- fsmonitor-ipc.c | 3 --- fsmonitor-settings.c | 3 --- t/t1093-virtualfilesystem.sh | 2 +- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/fsmonitor-ipc.c b/fsmonitor-ipc.c index 52a3392e16e465..ccc32d2a17ebe7 100644 --- a/fsmonitor-ipc.c +++ b/fsmonitor-ipc.c @@ -1,5 +1,4 @@ #include "cache.h" -#include "config.h" #include "fsmonitor.h" #include "simple-ipc.h" #include "fsmonitor-ipc.h" @@ -11,8 +10,6 @@ int fsmonitor_ipc__is_supported(void) { - if (git_config_get_virtualfilesystem()) - return 0; return 1; } diff --git a/fsmonitor-settings.c b/fsmonitor-settings.c index c900199423029c..1aa77b3193e1ff 100644 --- a/fsmonitor-settings.c +++ b/fsmonitor-settings.c @@ -127,9 +127,6 @@ static void lookup_fsmonitor_settings(struct repository *r) enum fsmonitor_mode fsm_settings__get_mode(struct repository *r) { - if (git_config_get_virtualfilesystem()) - return FSMONITOR_MODE_INCOMPATIBLE; - if (!r->settings.fsmonitor) lookup_fsmonitor_settings(r); diff --git a/t/t1093-virtualfilesystem.sh b/t/t1093-virtualfilesystem.sh index 0f43fc06e01571..0c6cf38fd347bf 100755 --- a/t/t1093-virtualfilesystem.sh +++ b/t/t1093-virtualfilesystem.sh @@ -368,7 +368,7 @@ test_expect_success 'folder with same prefix as file' ' test_cmp expected actual ' -test_expect_success 'virtualfilsystem hook disables built-in FSMonitor ' ' +test_expect_success MINGW,FSMONITOR_DAEMON 'virtualfilsystem hook disables built-in FSMonitor ' ' clean_repo && test_config core.usebuiltinfsmonitor true && write_script .git/hooks/virtualfilesystem <<-\EOF &&