From d569fecba1aeb6571e38161a7329c389354198eb Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Thu, 4 Oct 2018 18:10:21 -0400 Subject: [PATCH 1/4] mem_pool: add GIT_TRACE_MEMPOOL support Add tracing around initializing and discarding mempools. In discard report on the amount of memory unused in the current block to help tune setting the initial_size. Signed-off-by: Ben Peart --- mem-pool.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mem-pool.c b/mem-pool.c index a2841a4a9ad7e4..065389aaecf918 100644 --- a/mem-pool.c +++ b/mem-pool.c @@ -5,6 +5,7 @@ #include "cache.h" #include "mem-pool.h" +static struct trace_key trace_mem_pool = TRACE_KEY_INIT(MEMPOOL); #define BLOCK_GROWTH_SIZE 1024*1024 - sizeof(struct mp_block); /* @@ -48,12 +49,16 @@ void mem_pool_init(struct mem_pool **mem_pool, size_t initial_size) mem_pool_alloc_block(pool, initial_size, NULL); *mem_pool = pool; + trace_printf_key(&trace_mem_pool, "mem_pool (%p): init (%"PRIuMAX") initial size\n", + pool, (uintmax_t)initial_size); } void mem_pool_discard(struct mem_pool *mem_pool, int invalidate_memory) { struct mp_block *block, *block_to_free; + trace_printf_key(&trace_mem_pool, "mem_pool (%p): discard (%"PRIuMAX") unused\n", + mem_pool, (uintmax_t)(mem_pool->mp_block->end - mem_pool->mp_block->next_free)); block = mem_pool->mp_block; while (block) { From be35dedc8489ab7013ed9be0908614665d04f358 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Fri, 2 Nov 2018 11:19:10 -0400 Subject: [PATCH 2/4] fscache: fscache takes an initial size Update enable_fscache() to take an optional initial size parameter which is used to initialize the hashmap so that it can avoid having to rehash as additional entries are added. Add a separate disable_fscache() macro to make the code clearer and easier to read. Signed-off-by: Ben Peart --- builtin/add.c | 2 +- builtin/checkout.c | 4 ++-- builtin/commit.c | 4 ++-- compat/win32/fscache.c | 8 ++++++-- compat/win32/fscache.h | 5 +++-- fetch-pack.c | 4 ++-- git-compat-util.h | 4 ++++ preload-index.c | 4 ++-- read-cache.c | 4 ++-- 9 files changed, 24 insertions(+), 15 deletions(-) diff --git a/builtin/add.c b/builtin/add.c index f8ebf1c8e39753..1fab75ff6e0b2e 100644 --- a/builtin/add.c +++ b/builtin/add.c @@ -461,7 +461,7 @@ int cmd_add(int argc, const char **argv, const char *prefix) die_path_inside_submodule(&the_index, &pathspec); - enable_fscache(1); + enable_fscache(0); /* We do not really re-read the index but update the up-to-date flags */ preload_index(&the_index, &pathspec); diff --git a/builtin/checkout.c b/builtin/checkout.c index 07cc016e5cc3bd..ec0318231f09eb 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -360,7 +360,7 @@ static int checkout_paths(const struct checkout_opts *opts, state.istate = &the_index; enable_delayed_checkout(&state); - enable_fscache(1); + enable_fscache(active_nr); for (pos = 0; pos < active_nr; pos++) { struct cache_entry *ce = active_cache[pos]; if (ce->ce_flags & CE_MATCHED) { @@ -375,7 +375,7 @@ static int checkout_paths(const struct checkout_opts *opts, pos = skip_same_name(ce, pos) - 1; } } - enable_fscache(0); + disable_fscache(); errs |= finish_delayed_checkout(&state); if (write_locked_index(&the_index, &lock_file, COMMIT_LOCK)) diff --git a/builtin/commit.c b/builtin/commit.c index 8445d19dee4fe9..e98432a7525a33 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1376,7 +1376,7 @@ int cmd_status(int argc, const char **argv, const char *prefix) PATHSPEC_PREFER_FULL, prefix, argv); - enable_fscache(1); + enable_fscache(0); read_cache_preload(&s.pathspec); refresh_index(&the_index, REFRESH_QUIET|REFRESH_UNMERGED, &s.pathspec, NULL, NULL); @@ -1410,7 +1410,7 @@ int cmd_status(int argc, const char **argv, const char *prefix) s.prefix = prefix; wt_status_print(&s); - enable_fscache(0); + disable_fscache(); return 0; } diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index b81eb62e4f5dc1..96fe7e335c352f 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -401,7 +401,7 @@ static struct fsentry *fscache_get(struct fsentry *key) * Enables or disables the cache. Note that the cache is read-only, changes to * the working directory are NOT reflected in the cache while enabled. */ -int fscache_enable(int enable) +int fscache_enable(int enable, size_t initial_size) { int result; @@ -417,7 +417,11 @@ int fscache_enable(int enable) InitializeCriticalSection(&mutex); lstat_requests = opendir_requests = 0; fscache_misses = fscache_requests = 0; - hashmap_init(&map, (hashmap_cmp_fn) fsentry_cmp, NULL, 0); + /* + * avoid having to rehash by leaving room for the parent dirs. + * '4' was determined empirically by testing several repos + */ + hashmap_init(&map, (hashmap_cmp_fn) fsentry_cmp, NULL, initial_size * 4); initialized = 1; } diff --git a/compat/win32/fscache.h b/compat/win32/fscache.h index 2f06f8df97dcd0..d49c9381114da6 100644 --- a/compat/win32/fscache.h +++ b/compat/win32/fscache.h @@ -1,8 +1,9 @@ #ifndef FSCACHE_H #define FSCACHE_H -int fscache_enable(int enable); -#define enable_fscache(x) fscache_enable(x) +int fscache_enable(int enable, size_t initial_size); +#define enable_fscache(initial_size) fscache_enable(1, initial_size) +#define disable_fscache() fscache_enable(0, 0) int fscache_enabled(const char *path); #define is_fscache_enabled(path) fscache_enabled(path) diff --git a/fetch-pack.c b/fetch-pack.c index 43e0945da18021..d4f27db27cc024 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -678,7 +678,7 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator, save_commit_buffer = 0; - enable_fscache(1); + enable_fscache(0); for (ref = *refs; ref; ref = ref->next) { struct object *o; unsigned int flags = OBJECT_INFO_QUICK; @@ -708,7 +708,7 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator, cutoff = commit->date; } } - enable_fscache(0); + disable_fscache(); oidset_clear(&loose_oid_set); diff --git a/git-compat-util.h b/git-compat-util.h index 64a5d43a0e7860..e1b9e55683f204 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1259,6 +1259,10 @@ static inline int is_missing_file_error(int errno_) #define enable_fscache(x) /* noop */ #endif +#ifndef disable_fscache +#define disable_fscache() /* noop */ +#endif + #ifndef is_fscache_enabled #define is_fscache_enabled(path) (0) #endif diff --git a/preload-index.c b/preload-index.c index 6857573ce6de0a..af34c3dc59112a 100644 --- a/preload-index.c +++ b/preload-index.c @@ -91,7 +91,7 @@ void preload_index(struct index_state *index, const struct pathspec *pathspec) offset = 0; work = DIV_ROUND_UP(index->cache_nr, threads); memset(&data, 0, sizeof(data)); - enable_fscache(1); + enable_fscache(index->cache_nr); for (i = 0; i < threads; i++) { struct thread_data *p = data+i; p->index = index; @@ -109,7 +109,7 @@ void preload_index(struct index_state *index, const struct pathspec *pathspec) die("unable to join threaded lstat"); } trace_performance_since(start, "preload index"); - enable_fscache(0); + disable_fscache(); } #endif diff --git a/read-cache.c b/read-cache.c index 18b6fb42144ecb..6be61e5d0c887b 100644 --- a/read-cache.c +++ b/read-cache.c @@ -1483,7 +1483,7 @@ int refresh_index(struct index_state *istate, unsigned int flags, typechange_fmt = (in_porcelain ? "T\t%s\n" : "%s needs update\n"); added_fmt = (in_porcelain ? "A\t%s\n" : "%s needs update\n"); unmerged_fmt = (in_porcelain ? "U\t%s\n" : "%s: needs merge\n"); - enable_fscache(1); + enable_fscache(0); for (i = 0; i < istate->cache_nr; i++) { struct cache_entry *ce, *new_entry; int cache_errno = 0; @@ -1548,7 +1548,7 @@ int refresh_index(struct index_state *istate, unsigned int flags, replace_index_entry(istate, i, new_entry); } - enable_fscache(0); + disable_fscache(); trace_performance_since(start, "refresh index"); return has_errors; } From f1bf0982961bcd7b4c45e44564dbe7fde55328cd Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Thu, 4 Oct 2018 15:38:08 -0400 Subject: [PATCH 3/4] fscache: update fscache to be thread specific instead of global The threading model for fscache has been to have a single, global cache. This puts requirements on it to be thread safe so that callers like preload-index can call it from multiple threads. This was implemented with a single mutex and completion events which introduces contention between the calling threads. Simplify the threading model by making fscache thread specific. This allows us to remove the global mutex and synchronization events entirely and instead associate a fscache with every thread that requests one. This works well with the current multi-threading which divides the cache entries into blocks with a separate thread processing each block. At the end of each worker thread, if there is a fscache on the primary thread, merge the cached results from the worker into the primary thread cache. This enables us to reuse the cache later especially when scanning for untracked files. In testing, this reduced the time spent in preload_index() by about 25% and also reduced the CPU utilization significantly. On a repo with ~200K files, it reduced overall status times by ~12%. Signed-off-by: Ben Peart --- compat/win32/fscache.c | 292 +++++++++++++++++++++++++---------------- compat/win32/fscache.h | 22 +++- git-compat-util.h | 12 ++ preload-index.c | 7 +- 4 files changed, 213 insertions(+), 120 deletions(-) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 96fe7e335c352f..c3d76bf320d5d2 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -4,14 +4,24 @@ #include "fscache.h" #include "config.h" -static int initialized; -static volatile long enabled; -static struct hashmap map; +static volatile long initialized; +static DWORD dwTlsIndex; static CRITICAL_SECTION mutex; -static unsigned int lstat_requests; -static unsigned int opendir_requests; -static unsigned int fscache_requests; -static unsigned int fscache_misses; + +/* + * Store one fscache per thread to avoid thread contention and locking. + * This is ok because multi-threaded access is 1) uncommon and 2) always + * splitting up the cache entries across multiple threads so there isn't + * any overlap between threads anyway. + */ +struct fscache { + volatile long enabled; + struct hashmap map; + unsigned int lstat_requests; + unsigned int opendir_requests; + unsigned int fscache_requests; + unsigned int fscache_misses; +}; static struct trace_key trace_fscache = TRACE_KEY_INIT(FSCACHE); /* @@ -39,8 +49,6 @@ struct fsentry { union { /* Reference count of the directory listing. */ volatile long refcnt; - /* Handle to wait on the loading thread. */ - HANDLE hwait; struct { /* More stat members (only used for file entries). */ off64_t st_size; @@ -250,86 +258,63 @@ static struct fsentry *fsentry_create_list(const struct fsentry *dir, /* * Adds a directory listing to the cache. */ -static void fscache_add(struct fsentry *fse) +static void fscache_add(struct fscache *cache, struct fsentry *fse) { if (fse->list) fse = fse->list; for (; fse; fse = fse->next) - hashmap_add(&map, fse); + hashmap_add(&cache->map, fse); } /* * Clears the cache. */ -static void fscache_clear(void) +static void fscache_clear(struct fscache *cache) { - hashmap_free(&map, 1); - hashmap_init(&map, (hashmap_cmp_fn)fsentry_cmp, NULL, 0); - lstat_requests = opendir_requests = 0; - fscache_misses = fscache_requests = 0; + hashmap_free(&cache->map, 1); + hashmap_init(&cache->map, (hashmap_cmp_fn)fsentry_cmp, NULL, 0); + cache->lstat_requests = cache->opendir_requests = 0; + cache->fscache_misses = cache->fscache_requests = 0; } /* * Checks if the cache is enabled for the given path. */ -int fscache_enabled(const char *path) +static int do_fscache_enabled(struct fscache *cache, const char *path) { - return enabled > 0 && !is_absolute_path(path); + return cache->enabled > 0 && !is_absolute_path(path); } -/* - * Looks up a cache entry, waits if its being loaded by another thread. - * The mutex must be owned by the calling thread. - */ -static struct fsentry *fscache_get_wait(struct fsentry *key) +int fscache_enabled(const char *path) { - struct fsentry *fse = hashmap_get(&map, key, NULL); - - /* return if its a 'real' entry (future entries have refcnt == 0) */ - if (!fse || fse->list || fse->refcnt) - return fse; - - /* create an event and link our key to the future entry */ - key->hwait = CreateEvent(NULL, TRUE, FALSE, NULL); - key->next = fse->next; - fse->next = key; - - /* wait for the loading thread to signal us */ - LeaveCriticalSection(&mutex); - WaitForSingleObject(key->hwait, INFINITE); - CloseHandle(key->hwait); - EnterCriticalSection(&mutex); + struct fscache *cache = fscache_getcache(); - /* repeat cache lookup */ - return hashmap_get(&map, key, NULL); + return cache ? do_fscache_enabled(cache, path) : 0; } /* * Looks up or creates a cache entry for the specified key. */ -static struct fsentry *fscache_get(struct fsentry *key) +static struct fsentry *fscache_get(struct fscache *cache, struct fsentry *key) { - struct fsentry *fse, *future, *waiter; + struct fsentry *fse; int dir_not_found; - EnterCriticalSection(&mutex); - fscache_requests++; + cache->fscache_requests++; /* check if entry is in cache */ - fse = fscache_get_wait(key); + fse = hashmap_get(&cache->map, key, NULL); if (fse) { if (fse->st_mode) fsentry_addref(fse); else fse = NULL; /* non-existing directory */ - LeaveCriticalSection(&mutex); return fse; } /* if looking for a file, check if directory listing is in cache */ if (!fse && key->list) { - fse = fscache_get_wait(key->list); + fse = hashmap_get(&cache->map, key->list, NULL); if (fse) { - LeaveCriticalSection(&mutex); /* * dir entry without file entry, or dir does not * exist -> file doesn't exist @@ -339,25 +324,8 @@ static struct fsentry *fscache_get(struct fsentry *key) } } - /* add future entry to indicate that we're loading it */ - future = key->list ? key->list : key; - future->next = NULL; - future->refcnt = 0; - hashmap_add(&map, future); - - /* create the directory listing (outside mutex!) */ - LeaveCriticalSection(&mutex); - fse = fsentry_create_list(future, &dir_not_found); - EnterCriticalSection(&mutex); - - /* remove future entry and signal waiting threads */ - hashmap_remove(&map, future, NULL); - waiter = future->next; - while (waiter) { - HANDLE h = waiter->hwait; - waiter = waiter->next; - SetEvent(h); - } + /* create the directory listing */ + fse = fsentry_create_list(key->list ? key->list : key, &dir_not_found); /* leave on error (errno set by fsentry_create_list) */ if (!fse) { @@ -370,19 +338,18 @@ static struct fsentry *fscache_get(struct fsentry *key) fse = fsentry_alloc(key->list->list, key->list->name, key->list->len); fse->st_mode = 0; - hashmap_add(&map, fse); + hashmap_add(&cache->map, fse); } - LeaveCriticalSection(&mutex); return NULL; } /* add directory listing to the cache */ - fscache_misses++; - fscache_add(fse); + cache->fscache_misses++; + fscache_add(cache, fse); /* lookup file entry if requested (fse already points to directory) */ if (key->list) - fse = hashmap_get(&map, key, NULL); + fse = hashmap_get(&cache->map, key, NULL); if (fse && !fse->st_mode) fse = NULL; /* non-existing directory */ @@ -393,59 +360,102 @@ static struct fsentry *fscache_get(struct fsentry *key) else errno = ENOENT; - LeaveCriticalSection(&mutex); return fse; } /* - * Enables or disables the cache. Note that the cache is read-only, changes to + * Enables the cache. Note that the cache is read-only, changes to * the working directory are NOT reflected in the cache while enabled. */ -int fscache_enable(int enable, size_t initial_size) +int fscache_enable(size_t initial_size) { - int result; + int fscache; + struct fscache *cache; + int result = 0; + + /* allow the cache to be disabled entirely */ + fscache = git_env_bool("GIT_TEST_FSCACHE", -1); + if (fscache != -1) + core_fscache = fscache; + if (!core_fscache) + return 0; + /* + * refcount the global fscache initialization so that the + * opendir and lstat function pointers are redirected if + * any threads are using the fscache. + */ if (!initialized) { - int fscache = git_env_bool("GIT_TEST_FSCACHE", -1); - - /* allow the cache to be disabled entirely */ - if (fscache != -1) - core_fscache = fscache; - if (!core_fscache) - return 0; - InitializeCriticalSection(&mutex); - lstat_requests = opendir_requests = 0; - fscache_misses = fscache_requests = 0; + if (!dwTlsIndex) { + dwTlsIndex = TlsAlloc(); + if (dwTlsIndex == TLS_OUT_OF_INDEXES) + return 0; + } + + /* redirect opendir and lstat to the fscache implementations */ + opendir = fscache_opendir; + lstat = fscache_lstat; + } + InterlockedIncrement(&initialized); + + /* refcount the thread specific initialization */ + cache = fscache_getcache(); + if (cache) { + InterlockedIncrement(&cache->enabled); + } else { + cache = (struct fscache *)xcalloc(1, sizeof(*cache)); + cache->enabled = 1; /* * avoid having to rehash by leaving room for the parent dirs. * '4' was determined empirically by testing several repos */ - hashmap_init(&map, (hashmap_cmp_fn) fsentry_cmp, NULL, initial_size * 4); - initialized = 1; + hashmap_init(&cache->map, (hashmap_cmp_fn)fsentry_cmp, NULL, initial_size * 4); + if (!TlsSetValue(dwTlsIndex, cache)) + BUG("TlsSetValue error"); } - result = enable ? InterlockedIncrement(&enabled) - : InterlockedDecrement(&enabled); + trace_printf_key(&trace_fscache, "fscache: enable\n"); + return result; +} - if (enable && result == 1) { - /* redirect opendir and lstat to the fscache implementations */ - opendir = fscache_opendir; - lstat = fscache_lstat; - } else if (!enable && !result) { +/* + * Disables the cache. + */ +void fscache_disable(void) +{ + struct fscache *cache; + + if (!core_fscache) + return; + + /* update the thread specific fscache initialization */ + cache = fscache_getcache(); + if (!cache) + BUG("fscache_disable() called on a thread where fscache has not been initialized"); + if (!cache->enabled) + BUG("fscache_disable() called on an fscache that is already disabled"); + InterlockedDecrement(&cache->enabled); + if (!cache->enabled) { + TlsSetValue(dwTlsIndex, NULL); + trace_printf_key(&trace_fscache, "fscache_disable: lstat %u, opendir %u, " + "total requests/misses %u/%u\n", + cache->lstat_requests, cache->opendir_requests, + cache->fscache_requests, cache->fscache_misses); + fscache_clear(cache); + free(cache); + } + + /* update the global fscache initialization */ + InterlockedDecrement(&initialized); + if (!initialized) { /* reset opendir and lstat to the original implementations */ opendir = dirent_opendir; lstat = mingw_lstat; - EnterCriticalSection(&mutex); - trace_printf_key(&trace_fscache, "fscache: lstat %u, opendir %u, " - "total requests/misses %u/%u\n", - lstat_requests, opendir_requests, - fscache_requests, fscache_misses); - fscache_clear(); - LeaveCriticalSection(&mutex); } - trace_printf_key(&trace_fscache, "fscache: enable(%d)\n", enable); - return result; + + trace_printf_key(&trace_fscache, "fscache: disable\n"); + return; } /* @@ -453,10 +463,10 @@ int fscache_enable(int enable, size_t initial_size) */ void fscache_flush(void) { - if (enabled) { - EnterCriticalSection(&mutex); - fscache_clear(); - LeaveCriticalSection(&mutex); + struct fscache *cache = fscache_getcache(); + + if (cache && cache->enabled) { + fscache_clear(cache); } } @@ -468,11 +478,12 @@ int fscache_lstat(const char *filename, struct stat *st) { int dirlen, base, len; struct fsentry key[2], *fse; + struct fscache *cache = fscache_getcache(); - if (!fscache_enabled(filename)) + if (!cache || !do_fscache_enabled(cache, filename)) return mingw_lstat(filename, st); - lstat_requests++; + cache->lstat_requests++; /* split filename into path + name */ len = strlen(filename); if (len && is_dir_sep(filename[len - 1])) @@ -485,7 +496,7 @@ int fscache_lstat(const char *filename, struct stat *st) /* lookup entry for path + name in cache */ fsentry_init(key, NULL, filename, dirlen); fsentry_init(key + 1, key, filename + base, len - base); - fse = fscache_get(key + 1); + fse = fscache_get(cache, key + 1); if (!fse) return -1; @@ -549,11 +560,12 @@ DIR *fscache_opendir(const char *dirname) struct fsentry key, *list; fscache_DIR *dir; int len; + struct fscache *cache = fscache_getcache(); - if (!fscache_enabled(dirname)) + if (!cache || !do_fscache_enabled(cache, dirname)) return dirent_opendir(dirname); - opendir_requests++; + cache->opendir_requests++; /* prepare name (strip trailing '/', replace '.') */ len = strlen(dirname); if ((len == 1 && dirname[0] == '.') || @@ -562,7 +574,7 @@ DIR *fscache_opendir(const char *dirname) /* get directory listing from cache */ fsentry_init(&key, NULL, dirname, len); - list = fscache_get(&key); + list = fscache_get(cache, &key); if (!list) return NULL; @@ -573,3 +585,53 @@ DIR *fscache_opendir(const char *dirname) dir->pfsentry = list; return (DIR*) dir; } + +struct fscache *fscache_getcache(void) +{ + return (struct fscache *)TlsGetValue(dwTlsIndex); +} + +void fscache_merge(struct fscache *dest) +{ + struct hashmap_iter iter; + struct hashmap_entry *e; + struct fscache *cache = fscache_getcache(); + + /* + * Only do the merge if fscache was enabled and we have a dest + * cache to merge into. + */ + if (!dest) { + fscache_enable(0); + return; + } + if (!cache) + BUG("fscache_merge() called on a thread where fscache has not been initialized"); + + TlsSetValue(dwTlsIndex, NULL); + trace_printf_key(&trace_fscache, "fscache_merge: lstat %u, opendir %u, " + "total requests/misses %u/%u\n", + cache->lstat_requests, cache->opendir_requests, + cache->fscache_requests, cache->fscache_misses); + + /* + * This is only safe because the primary thread we're merging into + * isn't being used so the critical section only needs to prevent + * the the child threads from stomping on each other. + */ + EnterCriticalSection(&mutex); + + hashmap_iter_init(&cache->map, &iter); + while ((e = hashmap_iter_next(&iter))) + hashmap_add(&dest->map, e); + + dest->lstat_requests += cache->lstat_requests; + dest->opendir_requests += cache->opendir_requests; + dest->fscache_requests += cache->fscache_requests; + dest->fscache_misses += cache->fscache_misses; + LeaveCriticalSection(&mutex); + + free(cache); + + InterlockedDecrement(&initialized); +} diff --git a/compat/win32/fscache.h b/compat/win32/fscache.h index d49c9381114da6..2eb8bf3f5cfee8 100644 --- a/compat/win32/fscache.h +++ b/compat/win32/fscache.h @@ -1,9 +1,16 @@ #ifndef FSCACHE_H #define FSCACHE_H -int fscache_enable(int enable, size_t initial_size); -#define enable_fscache(initial_size) fscache_enable(1, initial_size) -#define disable_fscache() fscache_enable(0, 0) +/* + * The fscache is thread specific. enable_fscache() must be called + * for each thread where caching is desired. + */ + +int fscache_enable(size_t initial_size); +#define enable_fscache(initial_size) fscache_enable(initial_size) + +void fscache_disable(void); +#define disable_fscache() fscache_disable() int fscache_enabled(const char *path); #define is_fscache_enabled(path) fscache_enabled(path) @@ -14,4 +21,13 @@ void fscache_flush(void); DIR *fscache_opendir(const char *dir); int fscache_lstat(const char *file_name, struct stat *buf); +/* opaque fscache structure */ +struct fscache; + +struct fscache *fscache_getcache(void); +#define getcache_fscache() fscache_getcache() + +void fscache_merge(struct fscache *dest); +#define merge_fscache(dest) fscache_merge(dest) + #endif diff --git a/git-compat-util.h b/git-compat-util.h index e1b9e55683f204..2d1634611ef697 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1255,6 +1255,10 @@ static inline int is_missing_file_error(int errno_) * data or even file content without the need to synchronize with the file * system. */ + + /* opaque fscache structure */ +struct fscache; + #ifndef enable_fscache #define enable_fscache(x) /* noop */ #endif @@ -1271,6 +1275,14 @@ static inline int is_missing_file_error(int errno_) #define flush_fscache() /* noop */ #endif +#ifndef getcache_fscache +#define getcache_fscache() (NULL) /* noop */ +#endif + +#ifndef merge_fscache +#define merge_fscache(dest) /* noop */ +#endif + extern int cmd_main(int, const char **); /* diff --git a/preload-index.c b/preload-index.c index af34c3dc59112a..50956aa34c70b8 100644 --- a/preload-index.c +++ b/preload-index.c @@ -15,6 +15,8 @@ void preload_index(struct index_state *index, const struct pathspec *pathspec) #include +struct fscache *fscache; + /* * Mostly randomly chosen maximum thread counts: we * cap the parallelism to 20 threads, and we want @@ -43,6 +45,7 @@ static void *preload_thread(void *_data) if (nr + p->offset > index->cache_nr) nr = index->cache_nr - p->offset; + enable_fscache(nr); do { struct cache_entry *ce = *cep++; struct stat st; @@ -69,6 +72,7 @@ static void *preload_thread(void *_data) mark_fsmonitor_valid(ce); } while (--nr > 0); cache_def_clear(&cache); + merge_fscache(fscache); return NULL; } @@ -81,6 +85,7 @@ void preload_index(struct index_state *index, const struct pathspec *pathspec) if (!core_preload_index) return; + fscache = getcache_fscache(); threads = index->cache_nr / THREAD_COST; if ((index->cache_nr > 1) && (threads < 2) && getenv("GIT_FORCE_PRELOAD_TEST")) threads = 2; @@ -91,7 +96,6 @@ void preload_index(struct index_state *index, const struct pathspec *pathspec) offset = 0; work = DIV_ROUND_UP(index->cache_nr, threads); memset(&data, 0, sizeof(data)); - enable_fscache(index->cache_nr); for (i = 0; i < threads; i++) { struct thread_data *p = data+i; p->index = index; @@ -109,7 +113,6 @@ void preload_index(struct index_state *index, const struct pathspec *pathspec) die("unable to join threaded lstat"); } trace_performance_since(start, "preload index"); - disable_fscache(); } #endif From 1c2f6c80a77c2fbbd797c846e46486db2b72f471 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Fri, 2 Nov 2018 11:19:10 -0400 Subject: [PATCH 4/4] fscache: teach fscache to use mempool Now that the fscache is single threaded, take advantage of the mem_pool as the allocator to significantly reduce the cost of allocations and frees. With the reduced cost of free, in future patches, we can start freeing the fscache at the end of commands instead of just leaking it. Signed-off-by: Ben Peart --- compat/win32/fscache.c | 44 ++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index c3d76bf320d5d2..9c62a0a1429a44 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -3,6 +3,7 @@ #include "../win32.h" #include "fscache.h" #include "config.h" +#include "../../mem-pool.h" static volatile long initialized; static DWORD dwTlsIndex; @@ -17,6 +18,7 @@ static CRITICAL_SECTION mutex; struct fscache { volatile long enabled; struct hashmap map; + struct mem_pool *mem_pool; unsigned int lstat_requests; unsigned int opendir_requests; unsigned int fscache_requests; @@ -106,11 +108,11 @@ static void fsentry_init(struct fsentry *fse, struct fsentry *list, /* * Allocate an fsentry structure on the heap. */ -static struct fsentry *fsentry_alloc(struct fsentry *list, const char *name, +static struct fsentry *fsentry_alloc(struct fscache *cache, struct fsentry *list, const char *name, size_t len) { /* overallocate fsentry and copy the name to the end */ - struct fsentry *fse = xmalloc(sizeof(struct fsentry) + len + 1); + struct fsentry *fse = mem_pool_alloc(cache->mem_pool, sizeof(struct fsentry) + len + 1); char *nm = ((char*) fse) + sizeof(struct fsentry); memcpy(nm, name, len); nm[len] = 0; @@ -133,27 +135,20 @@ inline static void fsentry_addref(struct fsentry *fse) } /* - * Release the reference to an fsentry, frees the memory if its the last ref. + * Release the reference to an fsentry. */ static void fsentry_release(struct fsentry *fse) { if (fse->list) fse = fse->list; - if (InterlockedDecrement(&(fse->refcnt))) - return; - - while (fse) { - struct fsentry *next = fse->next; - free(fse); - fse = next; - } + InterlockedDecrement(&(fse->refcnt)); } /* * Allocate and initialize an fsentry from a WIN32_FIND_DATA structure. */ -static struct fsentry *fseentry_create_entry(struct fsentry *list, +static struct fsentry *fseentry_create_entry(struct fscache *cache, struct fsentry *list, const WIN32_FIND_DATAW *fdata) { char buf[MAX_PATH * 3]; @@ -161,7 +156,7 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, struct fsentry *fse; len = xwcstoutf(buf, fdata->cFileName, ARRAY_SIZE(buf)); - fse = fsentry_alloc(list, buf, len); + fse = fsentry_alloc(cache, list, buf, len); if (fdata->dwReserved0 == IO_REPARSE_TAG_SYMLINK && sizeof(buf) > (list ? list->len + 1 : 0) + fse->len + 1 && @@ -192,7 +187,7 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, * Dir should not contain trailing '/'. Use an empty string for the current * directory (not "."!). */ -static struct fsentry *fsentry_create_list(const struct fsentry *dir, +static struct fsentry *fsentry_create_list(struct fscache *cache, const struct fsentry *dir, int *dir_not_found) { wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ @@ -231,13 +226,13 @@ static struct fsentry *fsentry_create_list(const struct fsentry *dir, } /* allocate object to hold directory listing */ - list = fsentry_alloc(NULL, dir->name, dir->len); + list = fsentry_alloc(cache, NULL, dir->name, dir->len); list->st_mode = S_IFDIR; /* walk directory and build linked list of fsentry structures */ phead = &list->next; do { - *phead = fseentry_create_entry(list, &fdata); + *phead = fseentry_create_entry(cache, list, &fdata); phead = &(*phead)->next; } while (FindNextFileW(h, &fdata)); @@ -249,7 +244,7 @@ static struct fsentry *fsentry_create_list(const struct fsentry *dir, if (err == ERROR_NO_MORE_FILES) return list; - /* otherwise free the list and return error */ + /* otherwise release the list and return error */ fsentry_release(list); errno = err_win_to_posix(err); return NULL; @@ -272,7 +267,10 @@ static void fscache_add(struct fscache *cache, struct fsentry *fse) */ static void fscache_clear(struct fscache *cache) { - hashmap_free(&cache->map, 1); + mem_pool_discard(cache->mem_pool, 0); + cache->mem_pool = NULL; + mem_pool_init(&cache->mem_pool, 0); + hashmap_free(&cache->map, 0); hashmap_init(&cache->map, (hashmap_cmp_fn)fsentry_cmp, NULL, 0); cache->lstat_requests = cache->opendir_requests = 0; cache->fscache_misses = cache->fscache_requests = 0; @@ -325,7 +323,7 @@ static struct fsentry *fscache_get(struct fscache *cache, struct fsentry *key) } /* create the directory listing */ - fse = fsentry_create_list(key->list ? key->list : key, &dir_not_found); + fse = fsentry_create_list(cache, key->list ? key->list : key, &dir_not_found); /* leave on error (errno set by fsentry_create_list) */ if (!fse) { @@ -335,7 +333,7 @@ static struct fsentry *fscache_get(struct fscache *cache, struct fsentry *key) * empty, which for all practical matters is the same * thing as far as fscache is concerned). */ - fse = fsentry_alloc(key->list->list, + fse = fsentry_alloc(cache, key->list->list, key->list->name, key->list->len); fse->st_mode = 0; hashmap_add(&cache->map, fse); @@ -411,6 +409,7 @@ int fscache_enable(size_t initial_size) * '4' was determined empirically by testing several repos */ hashmap_init(&cache->map, (hashmap_cmp_fn)fsentry_cmp, NULL, initial_size * 4); + mem_pool_init(&cache->mem_pool, 0); if (!TlsSetValue(dwTlsIndex, cache)) BUG("TlsSetValue error"); } @@ -442,7 +441,8 @@ void fscache_disable(void) "total requests/misses %u/%u\n", cache->lstat_requests, cache->opendir_requests, cache->fscache_requests, cache->fscache_misses); - fscache_clear(cache); + mem_pool_discard(cache->mem_pool, 0); + hashmap_free(&cache->map, 0); free(cache); } @@ -625,6 +625,8 @@ void fscache_merge(struct fscache *dest) while ((e = hashmap_iter_next(&iter))) hashmap_add(&dest->map, e); + mem_pool_combine(dest->mem_pool, cache->mem_pool); + dest->lstat_requests += cache->lstat_requests; dest->opendir_requests += cache->opendir_requests; dest->fscache_requests += cache->fscache_requests;