Merge pull request #421: Sparse index: integrate with the `sparse-che…

…ckout` builtin

This integrates the `sparse-checkout` builtin with the sparse index. The tricky part here is that we need to partially expand the index when we are modifying the sparse-checkout definition.

Note that we modify the pattern list in a careful way: we create a `struct pattern_list` in-memory in `builtin/sparse-checkout.c` then apply those patterns to the index before writing the patterns to the sparse-checkout file. The `update_sparsity()` method does the work to assign the `SKIP_WORKTREE` bit appropriately, but this doesn't work if the files that are within the new sparse-checkout cone are still hidden behind a sparse directory.

The new `expand_to_pattern_list()` method does the hard work of expanding the sparse directories that are now within the new patterns. This expands only as far as needed, possibly creating new sparse directory entries.

This method does not contract existing files to sparse directories, and a big reason why is because of the check for ignored files as we delete those directories. The `clean_tracked_sparse_directories()` method is called after `update_sparsity()`, but we need to read the `A/B/.gitignore` file (or lack thereof) before we can delete `A/B/`. If we convert to sparse too quickly, then we lose this information and cause a full expansion.

Most of the correctness is handled by existing tests in `t1092`, but I add checks for `ensure_not_expanded` in some hopefully interesting cases.

As for performance, `git sparse-checkout set` can be slow if it needs to move a lot of files. However, no-op `git sparse-checkout set` (i.e. set the sparse-checkout cone to only include files at root, and do this on repeat) has these performance results on Linux in a monorepo with 2+ million files at `HEAD`:

```
Benchmark #1: baseline
  Time (mean ± σ):     10.465 s ±  0.018 s    [User: 9.885 s, System: 0.573 s]
  Range (min … max):   10.450 s … 10.497 s    5 runs
 
Benchmark #2: new code
  Time (mean ± σ):      68.9 ms ±   2.9 ms    [User: 45.8 ms, System: 17.1 ms]
  Range (min … max):    63.4 ms …  74.0 ms    41 runs
 
Summary
  'new code' ran
  151.89 ± 6.30 times faster than 'baseline'
```

dir.c

@@ -1400,46 +1400,16 @@ static struct path_pattern *last_matching_pattern_from_list(const char *pathname
 	return res;
 }
 
-/*
- * Scan the list of patterns to determine if the ordered list
- * of patterns matches on 'pathname'.
- *
- * Return 1 for a match, 0 for not matched and -1 for undecided.
- */
-enum pattern_match_result path_matches_pattern_list(
+enum pattern_match_result path_matches_cone_mode_pattern_list(
 				const char *pathname, int pathlen,
-				const char *basename, int *dtype,
-				struct pattern_list *pl,
-				struct index_state *istate)
+				struct pattern_list *pl)
 {
-	struct path_pattern *pattern;
 	struct strbuf parent_pathname = STRBUF_INIT;
 	int result = NOT_MATCHED;
 	size_t slash_pos;
 
-	/*
-	 * The virtual file system data is used to prevent git from traversing
-	 * any part of the tree that is not in the virtual file system.  Return
-	 * 1 to exclude the entry if it is not found in the virtual file system,
-	 * else fall through to the regular excludes logic as it may further exclude.
-	 */
-	if (*dtype == DT_UNKNOWN)
-		*dtype = resolve_dtype(DT_UNKNOWN, istate, pathname, pathlen);
-	if (is_excluded_from_virtualfilesystem(pathname, pathlen, *dtype) > 0)
-		return 1;
-
-	if (!pl->use_cone_patterns) {
-		pattern = last_matching_pattern_from_list(pathname, pathlen, basename,
-							dtype, pl, istate);
-		if (pattern) {
-			if (pattern->flags & PATTERN_FLAG_NEGATIVE)
-				return NOT_MATCHED;
-			else
-				return MATCHED;
-		}
-
-		return UNDECIDED;
-	}
+	if (!pl->use_cone_patterns)
+		BUG("path_matches_cone_mode_pattern_list requires cone mode patterns");
 
 	if (pl->full_cone)
 		return MATCHED;
@@ -1492,6 +1462,46 @@ enum pattern_match_result path_matches_pattern_list(
 	return result;
 }
 
+/*
+ * Scan the list of patterns to determine if the ordered list
+ * of patterns matches on 'pathname'.
+ *
+ * Return 1 for a match, 0 for not matched and -1 for undecided.
+ */
+enum pattern_match_result path_matches_pattern_list(
+				const char *pathname, int pathlen,
+				const char *basename, int *dtype,
+				struct pattern_list *pl,
+				struct index_state *istate)
+{
+	/*
+	 * The virtual file system data is used to prevent git from traversing
+	 * any part of the tree that is not in the virtual file system.  Return
+	 * 1 to exclude the entry if it is not found in the virtual file system,
+	 * else fall through to the regular excludes logic as it may further exclude.
+	 */
+	if (*dtype == DT_UNKNOWN)
+		*dtype = resolve_dtype(DT_UNKNOWN, istate, pathname, pathlen);
+	if (is_excluded_from_virtualfilesystem(pathname, pathlen, *dtype) > 0)
+		return 1;
+
+	if (!pl->use_cone_patterns) {
+		struct path_pattern *pattern = last_matching_pattern_from_list(
+							pathname, pathlen, basename,
+							dtype, pl, istate);
+		if (pattern) {
+			if (pattern->flags & PATTERN_FLAG_NEGATIVE)
+				return NOT_MATCHED;
+			else
+				return MATCHED;
+		}
+
+		return UNDECIDED;
+	}
+
+	return path_matches_cone_mode_pattern_list(pathname, pathlen, pl);
+}
+
 int init_sparse_checkout_patterns(struct index_state *istate)
 {
 	if (!core_apply_sparse_checkout)

dir.h

@@ -383,6 +383,15 @@ enum pattern_match_result {
 	MATCHED_RECURSIVE = 2,
 };
 
+/*
+ * Test if a given path is contained in the given pattern list.
+ *
+ * The given pattern list _must_ use cone mode patterns.
+ */
+enum pattern_match_result path_matches_cone_mode_pattern_list(
+				const char *pathname, int pathlen,
+				struct pattern_list *pl);
+
 /*
  * Scan the list of patterns to determine if the ordered list
  * of patterns matches on 'pathname'.