TODO: bundle-uri: fetch based on creationToken heuristic

Extend the existing testing of this heuristic case to include HTTP
transfers.

NEEDSWORK: the test isn't working to demonstrate that we don't need the
base bundle. (Also, we need 'git bundle fetch' for the current test
design)

Signed-off-by: Derrick Stolee <derrickstolee@github.com>

bundle-uri.c

@@ -417,6 +417,106 @@ static int download_bundle_to_file(struct remote_bundle_info *bundle, void *data
 	return fetch_bundle_uri_internal(ctx->r, bundle, ctx->depth + 1, ctx->list);
 }
 
+struct sorted_bundle_list {
+	struct remote_bundle_info **items;
+	size_t alloc;
+	size_t nr;
+};
+
+static int insert_bundle(struct remote_bundle_info *bundle, void *data)
+{
+	struct sorted_bundle_list *list = data;
+	list->items[list->nr++] = bundle;
+	return 0;
+}
+
+static int compare_creation_token(const void *va, const void *vb)
+{
+	const struct remote_bundle_info * const *a = va;
+	const struct remote_bundle_info * const *b = vb;
+
+	if ((*a)->creationToken > (*b)->creationToken)
+		return -1;
+	if ((*a)->creationToken < (*b)->creationToken)
+		return 1;
+	return 0;
+}
+
+static int fetch_bundles_by_token(struct repository *r,
+				  struct bundle_list *list)
+{
+	int cur;
+	int pop_or_push = 0;
+	struct bundle_list_context ctx = {
+		.r = r,
+		.list = list,
+		.mode = list->mode,
+	};
+	struct sorted_bundle_list sorted = {
+		.alloc = hashmap_get_size(&list->bundles),
+	};
+
+	ALLOC_ARRAY(sorted.items, sorted.alloc);
+
+	for_all_bundles_in_list(list, insert_bundle, &sorted);
+
+	QSORT(sorted.items, sorted.nr, compare_creation_token);
+
+	/*
+	 * Use a stack-based approach to download the bundles and attempt
+	 * to unbundle them in decreasing order by creation token. If we
+	 * fail to unbundle (after a successful download) then move to the
+	 * next non-downloaded bundle (push to the stack) and attempt
+	 * downloading. Once we succeed in applying a bundle, move to the
+	 * previous unapplied bundle (pop the stack) and attempt to unbundle
+	 * it again.
+	 */
+	cur = 0;
+	while (cur >= 0 && cur < sorted.nr) {
+		struct remote_bundle_info *bundle = sorted.items[cur];
+		if (!bundle->file.len) {
+			/* Not downloaded yet. Try downloading. */
+			if (download_bundle_to_file(bundle, &ctx)) {
+				/* Failure. Push to the stack. */
+				pop_or_push = 1;
+				goto stack_operation;
+			}
+		}
+
+		if (bundle->file.len && !bundle->unbundled) {
+			/*
+			 * This was downloaded, but not successfully
+			 * unbundled. Try unbundling again.
+			 */
+			if (unbundle_from_file(ctx.r, bundle->file.buf)) {
+				/* Failed to unbundle. Push to stack. */
+				pop_or_push = 1;
+			} else {
+				/* Succeeded in unbundle. Pop stack. */
+				pop_or_push = -1;
+			}
+		}
+
+		/*
+		 * Else case: downloaded and unbundled successfully.
+		 * Skip this by moving in the same direction as the
+		 * previous step.
+		 */
+
+stack_operation:
+		/* Move in the specified direction and repeat. */
+		cur += pop_or_push;
+	}
+
+	/*
+	 * We succeed if the loop terminates because 'cur' drops below
+	 * zero. The other case is that we terminate because 'cur'
+	 * reaches the end of the list, so we have a failure no matter
+	 * which bundles we apply from the list.
+	 */
+	return cur >= 0;
+}
+
 static int download_bundle_list(struct repository *r,
 				struct bundle_list *local_list,
 				struct bundle_list *global_list,
@@ -628,7 +728,11 @@ int fetch_bundle_list(struct repository *r, const char *uri, struct bundle_list
 	if ((result = download_bundle_list(r, list, &global_list, 0)))
 		goto cleanup;
 
-	result = unbundle_all_bundles(r, &global_list);
+	/* NEEDSWORK: what about global_list here? */
+	if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN)
+		result = fetch_bundles_by_token(r, list);
+	else
+		result = unbundle_all_bundles(r, &global_list);
 
 cleanup:
 	for_all_bundles_in_list(&global_list, unlink_bundle, NULL);

t/t5558-clone-bundle-uri.sh

@@ -230,6 +230,24 @@ test_expect_success 'clone bundle list (http, creationToken)' '
 	done
 '
 
+test_expect_success 'fetch bundle list (http, creationToken, incremental)' '
+	git clone --bare --no-local clone-from "$HTTPD_DOCUMENT_ROOT_PATH/clone-from.git" &&
+
+	git clone "$HTTPD_URL/smart/clone-from.git" clone-over-http &&
+	git -C fetch-http-3 fetch --bundle-uri="$HTTPD_URL/bundle-list" &&
+	git -C fetch-http-3 for-each-ref --format="%(refname)" "refs/bundles/*" >refs &&
+
+	# incremental fetch needs bundles 2, 3, and 4, but not 1.
+	# NEEDSWORK: we are still getting bundle 1?
+	cat >expect <<-\EOF &&
+	refs/bundles/base
+	refs/bundles/left
+	refs/bundles/merge
+	refs/bundles/right
+	EOF
+	test_cmp expect refs
+'
+
 # Do not add tests here unless they use the HTTP server, as they will
 # not run unless the HTTP dependencies exist.
 

t/t5601-clone.sh

@@ -849,6 +849,55 @@ test_expect_success 'auto-discover multiple bundles from HTTP clone' '
 	grep -f pattern trace.txt
 '
 
+test_expect_success 'auto-discover multiple bundles from HTTP clone: creationToken heuristic' '
+	test_when_finished rm -rf repo4 "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" &&
+
+	test_commit -C src newest &&
+	git -C src bundle create "$HTTPD_DOCUMENT_ROOT_PATH/newest.bundle" HEAD~1..HEAD &&
+	git clone --bare --no-local src "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" &&
+
+	git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" config \
+		uploadpack.advertiseBundleURIs true &&
+	git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" config \
+		bundle.version 1 &&
+	git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" config \
+		bundle.mode all &&
+	git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" config \
+		bundle.heuristic creationToken &&
+
+	git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" config \
+		bundle.everything.uri "$HTTPD_URL/everything.bundle" &&
+	git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" config \
+		bundle.everything.creationtoken 1 &&
+	git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" config \
+		bundle.new.uri "$HTTPD_URL/new.bundle" &&
+	git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" config \
+		bundle.new.creationtoken 2 &&
+	git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" config \
+		bundle.newest.uri "$HTTPD_URL/newest.bundle" &&
+	git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" config \
+		bundle.newest.creationtoken 3 &&
+
+	GIT_TEST_BUNDLE_URI=1 \
+	GIT_TRACE2_EVENT="$(pwd)/trace.txt" \
+		git -c protocol.version=2 clone \
+		$HTTPD_URL/smart/repo4.git repo3 &&
+
+	# We should fetch _both_ bundles
+	cat >pattern <<-EOF &&
+	"event":"child_start".*"argv":\["git-remote-https","origin","$HTTPD_URL/everything.bundle"\]
+	EOF
+	grep -f pattern trace.txt &&
+	cat >pattern <<-EOF &&
+	"event":"child_start".*"argv":\["git-remote-https","origin","$HTTPD_URL/new.bundle"\]
+	EOF
+	grep -f pattern trace.txt &&
+	cat >pattern <<-EOF &&
+	"event":"child_start".*"argv":\["git-remote-https","origin","$HTTPD_URL/newest.bundle"\]
+	EOF
+	grep -f pattern trace.txt
+'
+
 # DO NOT add non-httpd-specific tests here, because the last part of this
 # test script is only executed when httpd is available and enabled.