From 83be9bd88fdc02a1b8e68f35f40ddcd278b5755e Mon Sep 17 00:00:00 2001 From: Victoria Dye Date: Thu, 9 Sep 2021 10:30:27 -0400 Subject: [PATCH 1/5] checkout-index: expand sparse checkout compatibility tests Add tests to cover basic `checkout-index` cases related to sparse checkouts: files and folder, both inside and outside of sparse checkout definition. New tests will serve as a baseline for expected behavior when integrating `checkout-index` with the sparse index. Of note is the test demonstrating the behavior of `checkout-index --all`; creating files even outside the sparse checkout definition is somewhat unintuitive, and will cause significant performance issues when run with a sparse index. The goal of later changes will be to change this default behavior and introduce a `--sparse` flag to enable populating files outside the sparse definition. Signed-off-by: Victoria Dye --- t/t1092-sparse-checkout-compatibility.sh | 54 ++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index 4e16c2e156a787..ba914b83aae7b2 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -930,6 +930,60 @@ test_expect_success 'cherry-pick with conflicts' ' test_all_match test_must_fail git cherry-pick to-cherry-pick ' +test_expect_success 'checkout-index inside sparse definition' ' + init_repos && + + run_on_all rm -f deep/a && + test_all_match git checkout-index -- deep/a && + test_all_match git status --porcelain=v2 && + + echo test >>new-a && + run_on_all cp ../new-a a && + test_all_match test_must_fail git checkout-index -- a && + test_all_match git checkout-index -f -- a && + test_all_match git status --porcelain=v2 +' + +test_expect_success 'checkout-index outside sparse definition' ' + init_repos && + + # File does not exist on disk yet for sparse checkouts, so checkout-index + # succeeds without -f + test_sparse_match git checkout-index -- folder1/a && + test_cmp sparse-checkout/folder1/a sparse-index/folder1/a && + test_cmp sparse-checkout/folder1/a full-checkout/folder1/a && + + run_on_sparse rm -rf folder1 && + echo test >new-a && + run_on_sparse mkdir -p folder1 && + run_on_all cp ../new-a folder1/a && + + test_all_match test_must_fail git checkout-index -- folder1/a && + test_all_match git checkout-index -f -- folder1/a && + test_cmp sparse-checkout/folder1/a sparse-index/folder1/a && + test_cmp sparse-checkout/folder1/a full-checkout/folder1/a +' + +test_expect_success 'checkout-index with folders' ' + init_repos && + + # Inside checkout definition + test_all_match test_must_fail git checkout-index -f -- deep/ && + + # Outside checkout definition + test_all_match test_must_fail git checkout-index -f -- folder1/ +' + +# NEEDSWORK: even in sparse checkouts, checkout-index --all will create all +# files (even those outside the sparse definition) on disk. However, these files +# don't appear in the percentage of tracked files in git status. +test_expect_failure 'checkout-index --all' ' + init_repos && + + test_all_match git checkout-index --all && + test_sparse_match test_path_is_missing folder1 +' + test_expect_success 'clean' ' init_repos && From f940492d8353bd996710bb9e62d28148d39f0047 Mon Sep 17 00:00:00 2001 From: Victoria Dye Date: Fri, 10 Sep 2021 12:05:00 -0400 Subject: [PATCH 2/5] checkout-index: add `--sparse` option Change the default behavior of `checkout-index --all` for sparse checkouts to no longer refresh files outside the sparse checkout definition. The newly-added `--sparse` option, when used with `--all`, maintains the "old" behavior and checks out files outside the sparse checkout definition. Signed-off-by: Victoria Dye --- Documentation/git-checkout-index.txt | 10 ++++++++-- builtin/checkout-index.c | 12 ++++++++++-- builtin/stash.c | 2 +- t/t1092-sparse-checkout-compatibility.sh | 10 +++++----- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/Documentation/git-checkout-index.txt b/Documentation/git-checkout-index.txt index 4d33e7be0f5599..cf5d1de8424542 100644 --- a/Documentation/git-checkout-index.txt +++ b/Documentation/git-checkout-index.txt @@ -12,6 +12,7 @@ SYNOPSIS 'git checkout-index' [-u] [-q] [-a] [-f] [-n] [--prefix=] [--stage=|all] [--temp] + [--sparse] [-z] [--stdin] [--] [...] @@ -37,8 +38,9 @@ OPTIONS -a:: --all:: - checks out all files in the index. Cannot be used - together with explicit filenames. + checks out all files in the index, excluding those outside + any specified sparse checkout patterns (see `--sparse`). + Cannot be used together with explicit filenames. -n:: --no-create:: @@ -59,6 +61,10 @@ OPTIONS write the content to temporary files. The temporary name associations will be written to stdout. +--sparse:: + Refresh files outside of the sparse checkout boundary. May + only be used in conjunction with `--all`. + --stdin:: Instead of taking list of paths from the command line, read list of paths from the standard input. Paths are diff --git a/builtin/checkout-index.c b/builtin/checkout-index.c index e21620d964e5e8..e326cd12de8dbc 100644 --- a/builtin/checkout-index.c +++ b/builtin/checkout-index.c @@ -7,6 +7,7 @@ #define USE_THE_INDEX_COMPATIBILITY_MACROS #include "builtin.h" #include "config.h" +#include "dir.h" #include "lockfile.h" #include "quote.h" #include "cache-tree.h" @@ -116,7 +117,7 @@ static int checkout_file(const char *name, const char *prefix) return -1; } -static int checkout_all(const char *prefix, int prefix_length) +static int checkout_all(const char *prefix, int prefix_length, int include_sparse) { int i, errs = 0; struct cache_entry *last_ce = NULL; @@ -125,6 +126,8 @@ static int checkout_all(const char *prefix, int prefix_length) ensure_full_index(&the_index); for (i = 0; i < active_nr ; i++) { struct cache_entry *ce = active_cache[i]; + if (!include_sparse && !path_in_sparse_checkout(ce->name, &the_index)) + continue; if (ce_stage(ce) != checkout_stage && (CHECKOUT_ALL != checkout_stage || !ce_stage(ce))) continue; @@ -176,6 +179,7 @@ int cmd_checkout_index(int argc, const char **argv, const char *prefix) int i; struct lock_file lock_file = LOCK_INIT; int all = 0; + int include_sparse = 0; int read_from_stdin = 0; int prefix_length; int force = 0, quiet = 0, not_new = 0; @@ -185,6 +189,8 @@ int cmd_checkout_index(int argc, const char **argv, const char *prefix) struct option builtin_checkout_index_options[] = { OPT_BOOL('a', "all", &all, N_("check out all files in the index")), + OPT_BOOL(0, "sparse", &include_sparse, + N_("do not skip files outside the sparse checkout boundary")), OPT__FORCE(&force, N_("force overwrite of existing files"), 0), OPT__QUIET(&quiet, N_("no warning for existing files and files not in index")), @@ -247,6 +253,8 @@ int cmd_checkout_index(int argc, const char **argv, const char *prefix) if (all) die("git checkout-index: don't mix '--all' and explicit filenames"); + if (include_sparse) + die("git checkout-index: don't mix '--sparse' and explicit filenames"); if (read_from_stdin) die("git checkout-index: don't mix '--stdin' and explicit filenames"); p = prefix_path(prefix, prefix_length, arg); @@ -280,7 +288,7 @@ int cmd_checkout_index(int argc, const char **argv, const char *prefix) } if (all) - err |= checkout_all(prefix, prefix_length); + err |= checkout_all(prefix, prefix_length, include_sparse); if (pc_workers > 1) err |= run_parallel_checkout(&state, pc_workers, pc_threshold, diff --git a/builtin/stash.c b/builtin/stash.c index a0ccc8654dff70..ba936de2881a8c 100644 --- a/builtin/stash.c +++ b/builtin/stash.c @@ -365,7 +365,7 @@ static int restore_untracked(struct object_id *u_tree) child_process_init(&cp); cp.git_cmd = 1; - strvec_pushl(&cp.args, "checkout-index", "--all", NULL); + strvec_pushl(&cp.args, "checkout-index", "--all", "--sparse", NULL); strvec_pushf(&cp.env_array, "GIT_INDEX_FILE=%s", stash_index_path.buf); diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index ba914b83aae7b2..70de39a19b22df 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -974,14 +974,14 @@ test_expect_success 'checkout-index with folders' ' test_all_match test_must_fail git checkout-index -f -- folder1/ ' -# NEEDSWORK: even in sparse checkouts, checkout-index --all will create all -# files (even those outside the sparse definition) on disk. However, these files -# don't appear in the percentage of tracked files in git status. -test_expect_failure 'checkout-index --all' ' +test_expect_success 'checkout-index --all' ' init_repos && test_all_match git checkout-index --all && - test_sparse_match test_path_is_missing folder1 + test_sparse_match test_path_is_missing folder1 && + + test_all_match git checkout-index --sparse --all && + test_all_match test_path_exists folder1 ' test_expect_success 'clean' ' From a85e373e2563501c99d40e6e8c3ca82b837d304a Mon Sep 17 00:00:00 2001 From: Victoria Dye Date: Fri, 10 Sep 2021 12:42:02 -0400 Subject: [PATCH 3/5] checkout-index: integrate with sparse index Add repository settings to allow usage of the sparse index. In order to prevent unexpected errors when attempting to check out a sparse directory entry, `checkout_file` directly checks whether a found entry is a sparse directory and, if so, exits with an error. The test corresponding to this case now verifies the error message, intentionally differing from the non-sparse index scenarios. Signed-off-by: Victoria Dye --- builtin/checkout-index.c | 9 +++++++++ t/t1092-sparse-checkout-compatibility.sh | 10 +++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/builtin/checkout-index.c b/builtin/checkout-index.c index e326cd12de8dbc..dbc403afa43169 100644 --- a/builtin/checkout-index.c +++ b/builtin/checkout-index.c @@ -66,6 +66,7 @@ static int checkout_file(const char *name, const char *prefix) int namelen = strlen(name); int pos = cache_name_pos(name, namelen); int has_same_name = 0; + int is_file = 0; int did_checkout = 0; int errs = 0; @@ -79,6 +80,9 @@ static int checkout_file(const char *name, const char *prefix) break; has_same_name = 1; pos++; + if (S_ISSPARSEDIR(ce->ce_mode)) + break; + is_file = 1; if (ce_stage(ce) != checkout_stage && (CHECKOUT_ALL != checkout_stage || !ce_stage(ce))) continue; @@ -107,6 +111,8 @@ static int checkout_file(const char *name, const char *prefix) fprintf(stderr, "git checkout-index: %s ", name); if (!has_same_name) fprintf(stderr, "is not in the cache"); + else if (!is_file) + fprintf(stderr, "is a sparse directory"); else if (checkout_stage) fprintf(stderr, "does not exist at stage %d", checkout_stage); @@ -218,6 +224,9 @@ int cmd_checkout_index(int argc, const char **argv, const char *prefix) git_config(git_default_config, NULL); prefix_length = prefix ? strlen(prefix) : 0; + prepare_repo_settings(the_repository); + the_repository->settings.command_requires_full_index = 0; + if (read_cache() < 0) { die("invalid cache"); } diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index 70de39a19b22df..b6dd556c31e1ee 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -971,7 +971,14 @@ test_expect_success 'checkout-index with folders' ' test_all_match test_must_fail git checkout-index -f -- deep/ && # Outside checkout definition - test_all_match test_must_fail git checkout-index -f -- folder1/ + # Note: although all tests fail (as expected), the messaging differs. For + # non-sparse index checkouts, the error is that the "file" does not appear + # in the index; for sparse checkouts, the error is explicitly that the + # entry is a sparse directory. + run_on_all test_must_fail git checkout-index -f -- folder1/ && + test_cmp full-checkout-err sparse-checkout-err && + ! test_cmp full-checkout-err sparse-index-err && + grep "is a sparse directory" sparse-index-err ' test_expect_success 'checkout-index --all' ' @@ -1090,6 +1097,7 @@ test_expect_success 'sparse-index is not expanded' ' echo >>sparse-index/untracked.txt && ensure_not_expanded add . && + ensure_not_expanded checkout-index -f a && for ref in update-deep update-folder1 update-folder2 update-deep do echo >>sparse-index/README.md && From 688b8ecc9304829bc286752529e7e92b781ea09c Mon Sep 17 00:00:00 2001 From: Victoria Dye Date: Fri, 10 Sep 2021 12:50:53 -0400 Subject: [PATCH 4/5] checkout-index: improve `ensure_full_index` scope in `checkout_all` Update `checkout_all` to only run `ensure_full_index` when entries in sparse directories are needed (i.e., `--sparse` is specified). Signed-off-by: Victoria Dye --- builtin/checkout-index.c | 5 +++-- t/t1092-sparse-checkout-compatibility.sh | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/builtin/checkout-index.c b/builtin/checkout-index.c index dbc403afa43169..4ac566b5e22cb2 100644 --- a/builtin/checkout-index.c +++ b/builtin/checkout-index.c @@ -128,8 +128,9 @@ static int checkout_all(const char *prefix, int prefix_length, int include_spars int i, errs = 0; struct cache_entry *last_ce = NULL; - /* TODO: audit for interaction with sparse-index. */ - ensure_full_index(&the_index); + if (include_sparse) + ensure_full_index(&the_index); + for (i = 0; i < active_nr ; i++) { struct cache_entry *ce = active_cache[i]; if (!include_sparse && !path_in_sparse_checkout(ce->name, &the_index)) diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index b6dd556c31e1ee..73c97db10db13f 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -1098,6 +1098,7 @@ test_expect_success 'sparse-index is not expanded' ' ensure_not_expanded add . && ensure_not_expanded checkout-index -f a && + ensure_not_expanded checkout-index -f --all && for ref in update-deep update-folder1 update-folder2 update-deep do echo >>sparse-index/README.md && From cbaced7b7a25d44be00146ea5626625c73509755 Mon Sep 17 00:00:00 2001 From: Victoria Dye Date: Mon, 13 Sep 2021 18:48:10 -0400 Subject: [PATCH 5/5] checkout-index: add performance test for `checkout-index --all` `git checkout-index --all` is a subcommand of `git stash`, so it is helpful to verify the performance of that particular usage. Signed-off-by: Victoria Dye --- t/perf/p2000-sparse-operations.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/t/perf/p2000-sparse-operations.sh b/t/perf/p2000-sparse-operations.sh index af0be1e52b2eea..85a89f7a04a888 100755 --- a/t/perf/p2000-sparse-operations.sh +++ b/t/perf/p2000-sparse-operations.sh @@ -113,6 +113,7 @@ test_perf_on_all git checkout -f - test_perf_on_all git reset test_perf_on_all git reset --hard test_perf_on_all git reset -- does-not-exist +test_perf_on_all git checkout-index -f --all test_perf_on_all git update-index --add --remove test_perf_on_all git diff test_perf_on_all git diff --staged