Skip to content

Commit

Permalink
gvfs-helper: add prefetch .keep file for last packfile
Browse files Browse the repository at this point in the history 
Signed-off-by: Jeff Hostetler <jeffhost@microsoft.com>
  • Loading branch information
@jeffhostetler @dscho
jeffhostetler authored and dscho committed Aug 8, 2023
1 parent ec5b345 commit 6be1047
Show file tree
Hide file tree
Showing 2 changed files with 109 additions and 3 deletions.
83 changes: 80 additions & 3 deletions gvfs-helper.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1882,6 +1882,7 @@ static void my_run_index_pack(struct gh__request_params *params,

static void my_finalize_packfile(struct gh__request_params *params,
struct gh__response_status *status,
int b_keep,
const struct strbuf *temp_path_pack,
const struct strbuf *temp_path_idx,
struct strbuf *final_path_pack,
Expand All @@ -1901,6 +1902,21 @@ static void my_finalize_packfile(struct gh__request_params *params,
return;
}

if (b_keep) {
struct strbuf keep = STRBUF_INIT;
int fd_keep;

strbuf_addbuf(&keep, final_path_pack);
strbuf_strip_suffix(&keep, ".pack");
strbuf_addstr(&keep, ".keep");

fd_keep = xopen(keep.buf, O_WRONLY | O_CREAT | O_TRUNC, 0666);
if (fd_keep >= 0)
close(fd_keep);

strbuf_release(&keep);
}

if (params->result_list) {
struct strbuf result_msg = STRBUF_INIT;

Expand Down Expand Up @@ -1953,7 +1969,7 @@ static void install_packfile(struct gh__request_params *params,
create_final_packfile_pathnames("vfs", packfile_checksum.buf, NULL,
&final_path_pack, &final_path_idx,
&final_filename);
my_finalize_packfile(params, status,
my_finalize_packfile(params, status, 0,
&temp_path_pack, &temp_path_idx,
&final_path_pack, &final_path_idx,
&final_filename);
Expand Down Expand Up @@ -2049,6 +2065,12 @@ struct ph {

/*
* Extract the next packfile from the multipack.
* Install {.pack, .idx, .keep} set.
*
* Mark each successfully installed prefetch pack as .keep it as installed
* in case we have errors decoding/indexing later packs within the received
* multipart file. (A later pass can delete the unnecessary .keep files
* from this and any previous invocations.)
*/
static void extract_packfile_from_multipack(
struct gh__request_params *params,
Expand Down Expand Up @@ -2145,7 +2167,7 @@ static void extract_packfile_from_multipack(

} else {
/*
* Server send the .idx immediately after the .pack in the
* Server sent the .idx immediately after the .pack in the
* data stream. I'm tempted to verify it, but that defeats
* the purpose of having it cached...
*/
Expand All @@ -2167,7 +2189,7 @@ static void extract_packfile_from_multipack(
&final_path_pack, &final_path_idx,
&final_filename);

my_finalize_packfile(params, status,
my_finalize_packfile(params, status, 1,
&temp_path_pack, &temp_path_idx,
&final_path_pack, &final_path_idx,
&final_filename);
Expand All @@ -2182,6 +2204,56 @@ static void extract_packfile_from_multipack(
strbuf_release(&final_filename);
}

struct keep_files_data {
timestamp_t max_timestamp;
int pos_of_max;
struct string_list *keep_files;
};

static void cb_keep_files(const char *full_path, size_t full_path_len,
const char *file_path, void *void_data)
{
struct keep_files_data *data = void_data;
const char *val;
timestamp_t t;

/*
* We expect prefetch packfiles named like:
*
* prefetch-<seconds>-<checksum>.keep
*/
if (!skip_prefix(file_path, "prefetch-", &val))
return;
if (!ends_with(val, ".keep"))
return;

t = strtol(val, NULL, 10);
if (t > data->max_timestamp) {
data->pos_of_max = data->keep_files->nr;
data->max_timestamp = t;
}

string_list_append(data->keep_files, full_path);
}

static void delete_stale_keep_files(
struct gh__request_params *params,
struct gh__response_status *status)
{
struct string_list keep_files = STRING_LIST_INIT_DUP;
struct keep_files_data data = { 0, 0, &keep_files };
int k;

for_each_file_in_pack_dir(gh__global.buf_odb_path.buf,
cb_keep_files, &data);
for (k = 0; k < keep_files.nr; k++) {
if (k != data.pos_of_max)
unlink(keep_files.items[k].string);
}

string_list_clear(&keep_files, 0);
}

/*
* Cut apart the received multipart response into individual packfiles
* and install each one.
Expand All @@ -2200,6 +2272,7 @@ static void install_prefetch(struct gh__request_params *params,
unsigned short np;
unsigned short k;
int fd = -1;
int nr_installed = 0;

struct strbuf temp_path_mp = STRBUF_INIT;

Expand Down Expand Up @@ -2240,8 +2313,12 @@ static void install_prefetch(struct gh__request_params *params,
extract_packfile_from_multipack(params, status, fd, k);
if (status->ec != GH__ERROR_CODE__OK)
break;
nr_installed++;
}

if (nr_installed)
delete_stale_keep_files(params, status);

cleanup:
if (fd != -1)
close(fd);
Expand Down
29 changes: 29 additions & 0 deletions t/t5799-gvfs-helper.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,30 @@ verify_received_packfile_count () {
return 0
}

# Verify that we have exactly 1 prefetch .keep file.
# Optionally, verify that it has the given timestamp.
#
verify_prefetch_keeps () {
count=$(( $(ls -1 "$SHARED_CACHE_T1"/pack/prefetch-*.keep | wc -l) ))
if test $count -ne 1
then
echo "verify_prefetch_keep_file_count: found $count, expected 1."
return 1
fi

if test $# -eq 1
then
count=$(( $(ls -1 "$SHARED_CACHE_T1"/pack/prefetch-$1-*.keep | wc -l) ))
if test $count -ne 1
then
echo "verify_prefetch_keep_file_count: did not find expected keep file."
return 1
fi
fi

return 0
}

per_test_cleanup () {
stop_gvfs_protocol_server

Expand Down Expand Up @@ -643,6 +667,7 @@ test_expect_success 'basic: PREFETCH w/o arg gets all' '
# packfile.
#
verify_received_packfile_count 3 &&
verify_prefetch_keeps 1200000000 &&
stop_gvfs_protocol_server &&
verify_connection_count 1
Expand All @@ -664,6 +689,7 @@ test_expect_success 'basic: PREFETCH w/ arg' '
# packfile.
#
verify_received_packfile_count 2 &&
verify_prefetch_keeps 1200000000 &&
stop_gvfs_protocol_server &&
verify_connection_count 1
Expand All @@ -686,6 +712,7 @@ test_expect_success 'basic: PREFETCH mayhem no_prefetch_idx' '
# packfile.
#
verify_received_packfile_count 2 &&
verify_prefetch_keeps 1200000000 &&
stop_gvfs_protocol_server &&
verify_connection_count 1
Expand All @@ -707,6 +734,7 @@ test_expect_success 'basic: PREFETCH up-to-date' '
# packfile.
#
verify_received_packfile_count 2 &&
verify_prefetch_keeps 1200000000 &&
# Ask again for any packfiles newer than what we have cached locally.
#
Expand All @@ -720,6 +748,7 @@ test_expect_success 'basic: PREFETCH up-to-date' '
# packfile.
#
verify_received_packfile_count 0 &&
verify_prefetch_keeps 1200000000 &&
stop_gvfs_protocol_server &&
verify_connection_count 2
Expand Down

0 comments on commit 6be1047

Please sign in to comment.