diff --git a/Documentation/config/extensions.txt b/Documentation/config/extensions.txt index bccaec7a963679..6eb9a380ba9c0e 100644 --- a/Documentation/config/extensions.txt +++ b/Documentation/config/extensions.txt @@ -7,6 +7,31 @@ Note that this setting should only be set by linkgit:git-init[1] or linkgit:git-clone[1]. Trying to change it after initialization will not work and will produce hard-to-diagnose issues. +extensions.refFormat:: + Specify the reference storage mechanisms used by the repoitory as a + multi-valued list. The acceptable values are `files` and `packed`. + If not specified, the list of `files` and `packed` is assumed. It + is an error to specify this key unless `core.repositoryFormatVersion` + is 1. ++ +As new ref formats are added, Git commands may modify this list before and +after upgrading the on-disk reference storage files. The specific values +indicate the existence of different layers: ++ +* `files`: When present, references may be stored as "loose" reference files + in the `$GIT_DIR/refs/` directory. The name of the reference corresponds + to the filename after `$GIT_DIR` and the file contains an object ID as + a hexadecimal string. If a loose reference file exists, then its value + takes precedence over all other formats. ++ +* `packed`: When present, references may be stored as a group in a + `packed-refs` file in its version 1 format. When grouped with `"files"` + or provided on its own, this file is located at `$GIT_DIR/packed-refs`. + This file contains a list of distinct reference names, paired with their + object IDs. When combined with `files`, the `packed` format will only be + used to group multiple loose object files upon request via the + `git pack-refs` command or via the `pack-refs` maintenance task. + extensions.worktreeConfig:: If enabled, then worktrees will load config settings from the `$GIT_DIR/config.worktree` file in addition to the diff --git a/Documentation/config/index.txt b/Documentation/config/index.txt index 75f3a2d1054146..709ba72f62208b 100644 --- a/Documentation/config/index.txt +++ b/Documentation/config/index.txt @@ -30,3 +30,11 @@ index.version:: Specify the version with which new index files should be initialized. This does not affect existing repositories. If `feature.manyFiles` is enabled, then the default is 4. + +index.computeHash:: + When enabled, compute the hash of the index file as it is written + and store the hash at the end of the content. This is enabled by + default. ++ +If you disable `index.computHash`, then older Git clients may report that +your index is corrupt during `git fsck`. diff --git a/cache.h b/cache.h index 26ed03bd6de626..13e9c251ac3262 100644 --- a/cache.h +++ b/cache.h @@ -1155,6 +1155,8 @@ struct repository_format { int hash_algo; int sparse_index; char *work_tree; + int ref_format_count; + enum ref_format_flags ref_format; struct string_list unknown_extensions; struct string_list v1_only_extensions; }; diff --git a/csum-file.c b/csum-file.c index 59ef3398ca2b01..3243473c3d7e45 100644 --- a/csum-file.c +++ b/csum-file.c @@ -45,7 +45,8 @@ void hashflush(struct hashfile *f) unsigned offset = f->offset; if (offset) { - the_hash_algo->update_fn(&f->ctx, f->buffer, offset); + if (!f->skip_hash) + the_hash_algo->update_fn(&f->ctx, f->buffer, offset); flush(f, f->buffer, offset); f->offset = 0; } @@ -64,7 +65,12 @@ int finalize_hashfile(struct hashfile *f, unsigned char *result, int fd; hashflush(f); - the_hash_algo->final_fn(f->buffer, &f->ctx); + + if (f->skip_hash) + memset(f->buffer, 0, the_hash_algo->rawsz); + else + the_hash_algo->final_fn(f->buffer, &f->ctx); + if (result) hashcpy(result, f->buffer); if (flags & CSUM_HASH_IN_STREAM) @@ -108,7 +114,8 @@ void hashwrite(struct hashfile *f, const void *buf, unsigned int count) * the hashfile's buffer. In this block, * f->offset is necessarily zero. */ - the_hash_algo->update_fn(&f->ctx, buf, nr); + if (!f->skip_hash) + the_hash_algo->update_fn(&f->ctx, buf, nr); flush(f, buf, nr); } else { /* @@ -153,6 +160,7 @@ static struct hashfile *hashfd_internal(int fd, const char *name, f->tp = tp; f->name = name; f->do_crc = 0; + f->skip_hash = 0; the_hash_algo->init_fn(&f->ctx); f->buffer_len = buffer_len; diff --git a/csum-file.h b/csum-file.h index 0d29f528fbcb51..29468067f81880 100644 --- a/csum-file.h +++ b/csum-file.h @@ -20,6 +20,13 @@ struct hashfile { size_t buffer_len; unsigned char *buffer; unsigned char *check_buffer; + + /** + * If set to 1, skip_hash indicates that we should + * not actually compute the hash for this hashfile and + * instead only use it as a buffered write. + */ + unsigned int skip_hash; }; /* Checkpoint */ diff --git a/read-cache.c b/read-cache.c index 32024029274828..f24d96de4d360a 100644 --- a/read-cache.c +++ b/read-cache.c @@ -1817,6 +1817,8 @@ static int verify_hdr(const struct cache_header *hdr, unsigned long size) git_hash_ctx c; unsigned char hash[GIT_MAX_RAWSZ]; int hdr_version; + int all_zeroes = 1; + unsigned char *start, *end; if (hdr->hdr_signature != htonl(CACHE_SIGNATURE)) return error(_("bad signature 0x%08x"), hdr->hdr_signature); @@ -1827,10 +1829,23 @@ static int verify_hdr(const struct cache_header *hdr, unsigned long size) if (!verify_index_checksum) return 0; + end = (unsigned char *)hdr + size; + start = end - the_hash_algo->rawsz; + while (start < end) { + if (*start != 0) { + all_zeroes = 0; + break; + } + start++; + } + + if (all_zeroes) + return 0; + the_hash_algo->init_fn(&c); the_hash_algo->update_fn(&c, hdr, size - the_hash_algo->rawsz); the_hash_algo->final_fn(hash, &c); - if (!hasheq(hash, (unsigned char *)hdr + size - the_hash_algo->rawsz)) + if (!hasheq(hash, end - the_hash_algo->rawsz)) return error(_("bad index file sha1 signature")); return 0; } @@ -2917,9 +2932,14 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, int ieot_entries = 1; struct index_entry_offset_table *ieot = NULL; int nr, nr_threads; + int compute_hash; f = hashfd(tempfile->fd, tempfile->filename.buf); + if (!git_config_get_maybe_bool("index.computehash", &compute_hash) && + !compute_hash) + f->skip_hash = 1; + for (i = removed = extended = 0; i < entries; i++) { if (cache[i]->ce_flags & CE_REMOVE) removed++; diff --git a/refs.c b/refs.c index c89d558892569b..0cbe96bedf23b0 100644 --- a/refs.c +++ b/refs.c @@ -1956,6 +1956,15 @@ static struct ref_store *lookup_ref_store_map(struct hashmap *map, return entry ? entry->refs : NULL; } +static int add_ref_format_flags(enum ref_format_flags flags, int caps) { + if (flags & REF_FORMAT_FILES) + caps |= REF_STORE_FORMAT_FILES; + if (flags & REF_FORMAT_PACKED) + caps |= REF_STORE_FORMAT_PACKED; + + return caps; +} + /* * Create, record, and return a ref_store instance for the specified * gitdir. @@ -1965,9 +1974,17 @@ static struct ref_store *ref_store_init(struct repository *repo, unsigned int flags) { const char *be_name = "files"; - struct ref_storage_be *be = find_ref_storage_backend(be_name); + struct ref_storage_be *be; struct ref_store *refs; + flags = add_ref_format_flags(repo->ref_format, flags); + + if (!(flags & REF_STORE_FORMAT_FILES) && + (flags & REF_STORE_FORMAT_PACKED)) + be_name = "packed"; + + be = find_ref_storage_backend(be_name); + if (!be) BUG("reference backend %s is unknown", be_name); @@ -1983,7 +2000,8 @@ struct ref_store *get_main_ref_store(struct repository *r) if (!r->gitdir) BUG("attempting to get main_ref_store outside of repository"); - r->refs_private = ref_store_init(r, r->gitdir, REF_STORE_ALL_CAPS); + r->refs_private = ref_store_init(r, r->gitdir, + REF_STORE_ALL_CAPS); r->refs_private = maybe_debug_wrap_ref_store(r->gitdir, r->refs_private); return r->refs_private; } diff --git a/refs/files-backend.c b/refs/files-backend.c index e4009b3c421f5b..9848c010ea94d4 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -1207,6 +1207,12 @@ static int files_pack_refs(struct ref_store *ref_store, unsigned int flags) struct strbuf err = STRBUF_INIT; struct ref_transaction *transaction; + if (!packed_refs_enabled(refs->store_flags)) { + warning(_("refusing to create '%s' file because '%s' is not set"), + "packed-refs", "extensions.refFormat=packed"); + return -1; + } + transaction = ref_store_transaction_begin(refs->packed_ref_store, &err); if (!transaction) return -1; @@ -3282,7 +3288,7 @@ static int files_init_db(struct ref_store *ref_store, struct strbuf *err UNUSED) } struct ref_storage_be refs_be_files = { - .next = NULL, + .next = &refs_be_packed, .name = "files", .init = files_ref_store_create, .init_db = files_init_db, diff --git a/refs/packed-backend.c b/refs/packed-backend.c index 43cdb97f8b3775..120500050cf26f 100644 --- a/refs/packed-backend.c +++ b/refs/packed-backend.c @@ -478,6 +478,9 @@ static int load_contents(struct snapshot *snapshot) size_t size; ssize_t bytes_read; + if (!packed_refs_enabled(snapshot->refs->store_flags)) + return 0; + fd = open(snapshot->refs->path, O_RDONLY); if (fd < 0) { if (errno == ENOENT) { diff --git a/refs/refs-internal.h b/refs/refs-internal.h index 69f93b0e2ac9fa..a1900848a878e4 100644 --- a/refs/refs-internal.h +++ b/refs/refs-internal.h @@ -521,6 +521,14 @@ struct ref_store; REF_STORE_ODB | \ REF_STORE_MAIN) +#define REF_STORE_FORMAT_FILES (1 << 8) /* can use loose ref files */ +#define REF_STORE_FORMAT_PACKED (1 << 9) /* can use packed-refs file */ + +static inline int packed_refs_enabled(int flags) +{ + return flags & REF_STORE_FORMAT_PACKED; +} + /* * Initialize the ref_store for the specified gitdir. These functions * should call base_ref_store_init() to initialize the shared part of diff --git a/repository.c b/repository.c index 5d166b692c8aa8..96533fc76be3a6 100644 --- a/repository.c +++ b/repository.c @@ -182,6 +182,8 @@ int repo_init(struct repository *repo, repo->repository_format_partial_clone = format.partial_clone; format.partial_clone = NULL; + repo->ref_format = format.ref_format; + if (worktree) repo_set_worktree(repo, worktree); diff --git a/repository.h b/repository.h index 24316ac944edcd..5cfde4282c50c0 100644 --- a/repository.h +++ b/repository.h @@ -61,6 +61,11 @@ struct repo_path_cache { char *shallow; }; +enum ref_format_flags { + REF_FORMAT_FILES = (1 << 0), + REF_FORMAT_PACKED = (1 << 1), +}; + struct repository { /* Environment */ /* @@ -95,6 +100,7 @@ struct repository { * the ref object. */ struct ref_store *refs_private; + enum ref_format_flags ref_format; /* * Contains path to often used file names. diff --git a/setup.c b/setup.c index cefd5f63c4680f..a5e63479558a94 100644 --- a/setup.c +++ b/setup.c @@ -577,6 +577,16 @@ static enum extension_result handle_extension(const char *var, "extensions.objectformat", value); data->hash_algo = format; return EXTENSION_OK; + } else if (!strcmp(ext, "refformat")) { + if (!strcmp(value, "files")) + data->ref_format |= REF_FORMAT_FILES; + else if (!strcmp(value, "packed")) + data->ref_format |= REF_FORMAT_PACKED; + else + return error(_("invalid value for '%s': '%s'"), + "extensions.refFormat", value); + data->ref_format_count++; + return EXTENSION_OK; } return EXTENSION_UNKNOWN; } @@ -718,6 +728,11 @@ int read_repository_format(struct repository_format *format, const char *path) git_config_from_file(check_repo_format, path, format); if (format->version == -1) clear_repository_format(format); + + /* Set default ref_format if no extensions.refFormat exists. */ + if (!format->ref_format_count) + format->ref_format = REF_FORMAT_FILES | REF_FORMAT_PACKED; + return format->version; } @@ -1420,6 +1435,9 @@ int discover_git_directory(struct strbuf *commondir, candidate.partial_clone; candidate.partial_clone = NULL; + /* take ownership of candidate.ref_format */ + the_repository->ref_format = candidate.ref_format; + clear_repository_format(&candidate); return 0; } @@ -1556,6 +1574,8 @@ const char *setup_git_directory_gently(int *nongit_ok) the_repository->repository_format_partial_clone = repo_fmt.partial_clone; repo_fmt.partial_clone = NULL; + + the_repository->ref_format = repo_fmt.ref_format; } } /* @@ -1645,6 +1665,7 @@ void check_repository_format(struct repository_format *fmt) repo_set_hash_algo(the_repository, fmt->hash_algo); the_repository->repository_format_partial_clone = xstrdup_or_null(fmt->partial_clone); + the_repository->ref_format = fmt->ref_format; clear_repository_format(&repo_fmt); } diff --git a/t/t1600-index.sh b/t/t1600-index.sh index 010989f90e63f9..24ab90ca0478fe 100755 --- a/t/t1600-index.sh +++ b/t/t1600-index.sh @@ -103,4 +103,12 @@ test_expect_success 'index version config precedence' ' test_index_version 0 true 2 2 ' +test_expect_success 'index.computeHash config option' ' + ( + rm -f .git/index && + git -c index.computeHash=false add a && + git fsck + ) +' + test_done diff --git a/t/t3212-ref-formats.sh b/t/t3212-ref-formats.sh new file mode 100755 index 00000000000000..67aa65c116f642 --- /dev/null +++ b/t/t3212-ref-formats.sh @@ -0,0 +1,59 @@ +#!/bin/sh + +test_description='test across ref formats' + +. ./test-lib.sh + +test_expect_success 'extensions.refFormat requires core.repositoryFormatVersion=1' ' + test_when_finished rm -rf broken && + + # Force sha1 to ensure GIT_TEST_DEFAULT_HASH does + # not imply a value of core.repositoryFormatVersion. + git init --object-format=sha1 broken && + git -C broken config extensions.refFormat files && + test_must_fail git -C broken status 2>err && + grep "repo version is 0, but v1-only extension found" err +' + +test_expect_success 'invalid extensions.refFormat' ' + test_when_finished rm -rf broken && + git init broken && + git -C broken config core.repositoryFormatVersion 1 && + git -C broken config extensions.refFormat bogus && + test_must_fail git -C broken status 2>err && + grep "invalid value for '\''extensions.refFormat'\'': '\''bogus'\''" err +' + +test_expect_success 'extensions.refFormat=packed only' ' + git init only-packed && + ( + cd only-packed && + git config core.repositoryFormatVersion 1 && + git config extensions.refFormat packed && + test_commit A && + test_path_exists .git/packed-refs && + test_path_is_missing .git/refs/tags/A + ) +' + +test_expect_success 'extensions.refFormat=files only' ' + test_commit T && + git pack-refs --all && + git init only-loose && + ( + cd only-loose && + git config core.repositoryFormatVersion 1 && + git config extensions.refFormat files && + test_commit A && + test_commit B && + test_must_fail git pack-refs 2>err && + grep "refusing to create" err && + test_path_is_missing .git/packed-refs && + + # Refuse to parse a packed-refs file. + cp ../.git/packed-refs .git/packed-refs && + test_must_fail git rev-parse refs/tags/T + ) +' + +test_done