Skip to content

Commit

Permalink
fscache: remember not-found directories
Browse files Browse the repository at this point in the history
Teach FSCACHE to remember "not found" directories.

This is a performance optimization.

FSCACHE is a performance optimization available for Windows.  It
intercepts Posix-style lstat() calls into an in-memory directory
using FindFirst/FindNext.  It improves performance on Windows by
catching the first lstat() call in a directory, using FindFirst/
FindNext to read the list of files (and attribute data) for the
entire directory into the cache, and short-cut subsequent lstat()
calls in the same directory.  This gives a major performance
boost on Windows.

However, it does not remember "not found" directories.  When STATUS
runs and there are missing directories, the lstat() interception
fails to find the parent directory and simply return ENOENT for the
file -- it does not remember that the FindFirst on the directory
failed. Thus subsequent lstat() calls in the same directory, each
re-attempt the FindFirst.  This completely defeats any performance
gains.

This can be seen by doing a sparse-checkout on a large repo and
then doing a read-tree to reset the skip-worktree bits and then
running status.

This change reduced status times for my very large repo by 60%.

Signed-off-by: Jeff Hostetler <jeffhost@microsoft.com>
  • Loading branch information
jeffhostetler committed Jan 24, 2017
1 parent 0217bed commit 8544ec3
Showing 1 changed file with 30 additions and 4 deletions.
34 changes: 30 additions & 4 deletions compat/win32/fscache.c
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,8 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list,
* Dir should not contain trailing '/'. Use an empty string for the current
* directory (not "."!).
*/
static struct fsentry *fsentry_create_list(const struct fsentry *dir)
static struct fsentry *fsentry_create_list(const struct fsentry *dir,
int *dir_not_found)
{
wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */
WIN32_FIND_DATAW fdata;
Expand All @@ -173,6 +174,8 @@ static struct fsentry *fsentry_create_list(const struct fsentry *dir)
struct fsentry *list, **phead;
DWORD err;

*dir_not_found = 0;

/* convert name to UTF-16 and check length */
if ((wlen = xutftowcs_path_ex(pattern, dir->name, MAX_LONG_PATH,
dir->len, MAX_PATH - 2, core_long_paths)) < 0)
Expand All @@ -191,6 +194,7 @@ static struct fsentry *fsentry_create_list(const struct fsentry *dir)
h = FindFirstFileW(pattern, &fdata);
if (h == INVALID_HANDLE_VALUE) {
err = GetLastError();
*dir_not_found = 1; /* or empty directory */
errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err);
trace_printf_key(&trace_fscache, "fscache: error(%d) '%.*s'\n",
errno, dir->len, dir->name);
Expand All @@ -199,6 +203,7 @@ static struct fsentry *fsentry_create_list(const struct fsentry *dir)

/* allocate object to hold directory listing */
list = fsentry_alloc(NULL, dir->name, dir->len);
list->st_mode = S_IFDIR;

/* walk directory and build linked list of fsentry structures */
phead = &list->next;
Expand Down Expand Up @@ -299,12 +304,16 @@ static struct fsentry *fscache_get_wait(struct fsentry *key)
static struct fsentry *fscache_get(struct fsentry *key)
{
struct fsentry *fse, *future, *waiter;
int dir_not_found;

EnterCriticalSection(&mutex);
/* check if entry is in cache */
fse = fscache_get_wait(key);
if (fse) {
fsentry_addref(fse);
if (fse->st_mode)
fsentry_addref(fse);
else
fse = NULL; /* non-existing directory */
LeaveCriticalSection(&mutex);
return fse;
}
Expand All @@ -313,7 +322,10 @@ static struct fsentry *fscache_get(struct fsentry *key)
fse = fscache_get_wait(key->list);
if (fse) {
LeaveCriticalSection(&mutex);
/* dir entry without file entry -> file doesn't exist */
/*
* dir entry without file entry, or dir does not
* exist -> file doesn't exist
*/
errno = ENOENT;
return NULL;
}
Expand All @@ -327,7 +339,7 @@ static struct fsentry *fscache_get(struct fsentry *key)

/* create the directory listing (outside mutex!) */
LeaveCriticalSection(&mutex);
fse = fsentry_create_list(future);
fse = fsentry_create_list(future, &dir_not_found);
EnterCriticalSection(&mutex);

/* remove future entry and signal waiting threads */
Expand All @@ -341,6 +353,17 @@ static struct fsentry *fscache_get(struct fsentry *key)

/* leave on error (errno set by fsentry_create_list) */
if (!fse) {
if (dir_not_found && key->list) {
/*
* Record that the directory does not exist (or is
* empty, which for all practical matters is the same
* thing as far as fscache is concerned).
*/
fse = fsentry_alloc(key->list->list,
key->list->name, key->list->len);
fse->st_mode = 0;
hashmap_add(&map, fse);
}
LeaveCriticalSection(&mutex);
return NULL;
}
Expand All @@ -352,6 +375,9 @@ static struct fsentry *fscache_get(struct fsentry *key)
if (key->list)
fse = hashmap_get(&map, key, NULL);

if (fse && !fse->st_mode)
fse = NULL; /* non-existing directory */

/* return entry or ENOENT */
if (fse)
fsentry_addref(fse);
Expand Down

0 comments on commit 8544ec3

Please sign in to comment.