From 4efbcaa6fd21bdeaca14b3d49fbeabbf01bc362a Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 21 Jun 2022 17:54:35 -0400 Subject: [PATCH] bundle-uri: fetch based on creationToken heuristic Extend the existing testing of this heuristic case to include HTTP transfers. Signed-off-by: Derrick Stolee --- bundle-uri.c | 106 +++++++++++++++++++++++++++++++++++- t/t5558-fetch-bundle-uri.sh | 18 ++++++ t/t5601-clone.sh | 49 +++++++++++++++++ 3 files changed, 172 insertions(+), 1 deletion(-) diff --git a/bundle-uri.c b/bundle-uri.c index 5426401c9ef90d..ca55848ddc6a98 100644 --- a/bundle-uri.c +++ b/bundle-uri.c @@ -407,6 +407,106 @@ static int download_bundle_to_file(struct remote_bundle_info *bundle, void *data return fetch_bundle_uri_internal(ctx->r, bundle, ctx->depth + 1, ctx->list); } +struct sorted_bundle_list { + struct remote_bundle_info **items; + size_t alloc; + size_t nr; +}; + +static int insert_bundle(struct remote_bundle_info *bundle, void *data) +{ + struct sorted_bundle_list *list = data; + list->items[list->nr++] = bundle; + return 0; +} + +static int compare_creation_token(const void *va, const void *vb) +{ + const struct remote_bundle_info * const *a = va; + const struct remote_bundle_info * const *b = vb; + + if ((*a)->creationToken > (*b)->creationToken) + return -1; + if ((*a)->creationToken < (*b)->creationToken) + return 1; + return 0; +} + +static int fetch_bundles_by_token(struct repository *r, + struct bundle_list *list) +{ + int cur; + int pop_or_push = 0; + struct bundle_list_context ctx = { + .r = r, + .list = list, + .mode = list->mode, + }; + struct sorted_bundle_list sorted = { + .alloc = hashmap_get_size(&list->bundles), + }; + + ALLOC_ARRAY(sorted.items, sorted.alloc); + + for_all_bundles_in_list(list, insert_bundle, &sorted); + + QSORT(sorted.items, sorted.nr, compare_creation_token); + + /* + * Use a stack-based approach to download the bundles and attempt + * to unbundle them in decreasing order by creation token. If we + * fail to unbundle (after a successful download) then move to the + * next non-downloaded bundle (push to the stack) and attempt + * downloading. Once we succeed in applying a bundle, move to the + * previous unapplied bundle (pop the stack) and attempt to unbundle + * it again. + */ + cur = 0; + while (cur >= 0 && cur < sorted.nr) { + struct remote_bundle_info *bundle = sorted.items[cur]; + if (!bundle->file.len) { + /* Not downloaded yet. Try downloading. */ + if (download_bundle_to_file(bundle, &ctx)) { + /* Failure. Push to the stack. */ + pop_or_push = 1; + goto stack_operation; + } + } + + if (bundle->file.len && !bundle->unbundled) { + /* + * This was downloaded, but not successfully + * unbundled. Try unbundling again. + */ + if (unbundle_from_file(ctx.r, bundle->file.buf)) { + /* Failed to unbundle. Push to stack. */ + pop_or_push = 1; + } else { + /* Succeeded in unbundle. Pop stack. */ + pop_or_push = -1; + } + } + + /* + * Else case: downloaded and unbundled successfully. + * Skip this by moving in the same direction as the + * previous step. + */ + +stack_operation: + /* Move in the specified direction and repeat. */ + cur += pop_or_push; + } + + /* + * We succeed if the loop terminates because 'cur' drops below + * zero. The other case is that we terminate because 'cur' + * reaches the end of the list, so we have a failure no matter + * which bundles we apply from the list. + */ + return cur >= 0; +} + static int download_bundle_list(struct repository *r, struct bundle_list *local_list, struct bundle_list *global_list, @@ -611,7 +711,11 @@ int fetch_bundle_list(struct repository *r, const char *uri, struct bundle_list if ((result = download_bundle_list(r, list, &global_list, 0))) goto cleanup; - result = unbundle_all_bundles(r, &global_list); + /* NEEDSWORK: what about global_list here? */ + if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN) + result = fetch_bundles_by_token(r, list); + else + result = unbundle_all_bundles(r, &global_list); cleanup: for_all_bundles_in_list(&global_list, unlink_bundle, NULL); diff --git a/t/t5558-fetch-bundle-uri.sh b/t/t5558-fetch-bundle-uri.sh index 66dee998f50676..bdec541f6f5ae3 100755 --- a/t/t5558-fetch-bundle-uri.sh +++ b/t/t5558-fetch-bundle-uri.sh @@ -265,6 +265,24 @@ test_expect_success 'fetch bundle list (http, creationToken)' ' done ' +test_expect_success 'fetch bundle list (http, creationToken, incremental)' ' + git clone --bare --no-local fetch-from "$HTTPD_DOCUMENT_ROOT_PATH/fetch.git" && + + git init fetch-http-3 && + git -C fetch-http-3 fetch "$HTTPD_URL/smart/fetch.git" \ + refs/heads/right:refs/heads/right --no-tags && + git -C fetch-http-3 fetch --bundle-uri="$HTTPD_URL/bundle-list" && + git -C fetch-http-3 for-each-ref --format="%(refname)" "refs/bundles/*" >refs && + + # incremental fetch needs bundles 2, 3, and 4, but not 1. + cat >expect <<-\EOF && + refs/bundles/left + refs/bundles/merge + refs/bundles/right + EOF + test_cmp expect refs +' + # Do not add tests here unless they use the HTTP server, as they will # not run unless the HTTP dependencies exist. diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh index 8ef3b2257b44bb..bc645dba79753d 100755 --- a/t/t5601-clone.sh +++ b/t/t5601-clone.sh @@ -849,6 +849,55 @@ test_expect_success 'auto-discover multiple bundles from HTTP clone' ' grep -f pattern trace.txt ' +test_expect_success 'auto-discover multiple bundles from HTTP clone: creationToken heuristic' ' + test_when_finished rm -rf repo4 "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" && + + test_commit -C src newest && + git -C src bundle create "$HTTPD_DOCUMENT_ROOT_PATH/newest.bundle" HEAD~1..HEAD && + git clone --bare --no-local src "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" && + + git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" config \ + uploadpack.advertiseBundleURIs true && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" config \ + bundle.version 1 && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" config \ + bundle.mode all && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" config \ + bundle.heuristic creationToken && + + git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" config \ + bundle.everything.uri "$HTTPD_URL/everything.bundle" && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" config \ + bundle.everything.creationtoken 1 && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" config \ + bundle.new.uri "$HTTPD_URL/new.bundle" && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" config \ + bundle.new.creationtoken 2 && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" config \ + bundle.newest.uri "$HTTPD_URL/newest.bundle" && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" config \ + bundle.newest.creationtoken 3 && + + GIT_TEST_BUNDLE_URI=1 \ + GIT_TRACE2_EVENT="$(pwd)/trace.txt" \ + git -c protocol.version=2 clone \ + $HTTPD_URL/smart/repo4.git repo3 && + + # We should fetch _both_ bundles + cat >pattern <<-EOF && + "event":"child_start".*"argv":\["git-remote-https","origin","$HTTPD_URL/everything.bundle"\] + EOF + grep -f pattern trace.txt && + cat >pattern <<-EOF && + "event":"child_start".*"argv":\["git-remote-https","origin","$HTTPD_URL/new.bundle"\] + EOF + grep -f pattern trace.txt && + cat >pattern <<-EOF && + "event":"child_start".*"argv":\["git-remote-https","origin","$HTTPD_URL/newest.bundle"\] + EOF + grep -f pattern trace.txt +' + # DO NOT add non-httpd-specific tests here, because the last part of this # test script is only executed when httpd is available and enabled.