From 268fce8a3e49dd86358a491873905b6635dd9157 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 16 Nov 2021 18:08:24 +0100 Subject: [PATCH 01/35] scalar: allow installing the command Now that we implemented Scalar's core functionality, let's offer users the option to install the command via `make -C contrib/scalar install`. Signed-off-by: Johannes Schindelin --- contrib/scalar/Makefile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/contrib/scalar/Makefile b/contrib/scalar/Makefile index 37f283f35d74d6..3944efe2caa23d 100644 --- a/contrib/scalar/Makefile +++ b/contrib/scalar/Makefile @@ -32,4 +32,14 @@ clean: test: all $(MAKE) -C t -.PHONY: $(GITLIBS) all clean test FORCE +INSTALL = install +prefix = $(HOME) +bindir = $(prefix)/bin +DESTDIR_SQ = $(subst ','\'',$(DESTDIR)) +bindir_SQ = $(subst ','\'',$(bindir)) + +install: + $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) scalar$(X) '$(DESTDIR_SQ)$(bindir_SQ)' + +.PHONY: $(GITLIBS) all clean test install FORCE From 5555fd5e454f705cb18bd9ac09e0f071d7ca40f0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 3 May 2021 15:34:03 +0200 Subject: [PATCH 02/35] scalar: allow building and installing the documentation Now that Scalar is reasonably complete, we are comfortable having users install its documentation, too. Signed-off-by: Johannes Schindelin --- contrib/scalar/.gitignore | 3 +++ contrib/scalar/Makefile | 26 +++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/contrib/scalar/.gitignore b/contrib/scalar/.gitignore index ff3d47e84d0436..00441073f59cf5 100644 --- a/contrib/scalar/.gitignore +++ b/contrib/scalar/.gitignore @@ -1,2 +1,5 @@ +/*.xml +/*.1 +/*.html /*.exe /scalar diff --git a/contrib/scalar/Makefile b/contrib/scalar/Makefile index 3944efe2caa23d..df5ef37af94a82 100644 --- a/contrib/scalar/Makefile +++ b/contrib/scalar/Makefile @@ -21,6 +21,7 @@ $(TARGETS): $(GITLIBS) scalar.c clean: $(RM) $(TARGETS) ../../bin-wrappers/scalar + $(RM) scalar.1 scalar.html scalar.xml ../../bin-wrappers/scalar: ../../wrap-for-bin.sh Makefile @mkdir -p ../../bin-wrappers @@ -35,11 +36,34 @@ test: all INSTALL = install prefix = $(HOME) bindir = $(prefix)/bin +mandir ?= $(prefix)/share/man +man1dir = $(mandir)/man1 +htmldir ?= $(prefix)/share/doc/git-doc DESTDIR_SQ = $(subst ','\'',$(DESTDIR)) bindir_SQ = $(subst ','\'',$(bindir)) +man1dir_SQ = $(subst ','\'',$(man1dir)) +htmldir_SQ = $(subst ','\'',$(htmldir)) install: $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(bindir_SQ)' $(INSTALL) scalar$(X) '$(DESTDIR_SQ)$(bindir_SQ)' -.PHONY: $(GITLIBS) all clean test install FORCE +docs: scalar.html scalar.1 + +scalar.html: | scalar.1 # prevent them from trying to build `doc.dep` in parallel + +scalar.html scalar.1: scalar.txt + $(QUIET_SUBDIR0)../../Documentation$(QUIET_SUBDIR1) \ + MAN_TXT=../contrib/scalar/scalar.txt \ + ../contrib/scalar/$@ + $(QUIET)test scalar.1 != "$@" || mv ../../Documentation/$@ . + +install-doc: scalar.1 + $(INSTALL) -d -m 755 $(DESTDIR_SQ)$(man1dir_SQ) + $(INSTALL) -m 644 $^ $(DESTDIR_SQ)$(man1dir_SQ) + +install-html: scalar.html + $(INSTALL) -d -m 755 $(DESTDIR_SQ)$(htmldir_SQ) + $(INSTALL) -m 644 $^ $(DESTDIR_SQ)$(htmldir_SQ) + +.PHONY: $(GITLIBS) all clean test docs install install-doc install-html FORCE From b3e9863096fd4f292f554fb0bd40595aa94ada82 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 16 Nov 2021 17:45:18 +0100 Subject: [PATCH 03/35] cmake: optionally build `scalar`, too Unlike the `Makefile`-based build, the CMake configuration unfortunately does not let us easily encapsulate Scalar's build definition in the `contrib/scalar/` subdirectory: The `scalar` executable needs to link in `libgit.a` and `common-main.o`, for example. Also, `scalar.c` includes Git's header files, which means that `scalar.c` needs to be compiled with the very same flags as `libgit.a` lest `scalar.o` and `libgit.a` have different ideas of, say, `platform_SHA_CTX`, which would naturally lead to memory corruption, crashes and quite tricky debugging (talking from experience). To alleviate that lack of encapsulation somewhat, we guard the Scalar parts in `CMakeLists.txt` via the `INCLUDE_SCALAR` environment variable. This not only allows the CMake-based build to exclude Scalar by default, but also gives better visual cues as to which sections are related to Scalar. Signed-off-by: Johannes Schindelin --- contrib/buildsystems/CMakeLists.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 199410c4ac24d0..d1b0fdf94a3348 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -810,6 +810,13 @@ if(CURL_FOUND) endif() endif() +if(DEFINED ENV{INCLUDE_SCALAR} AND NOT ENV{INCLUDE_SCALAR} STREQUAL "") + add_executable(scalar ${CMAKE_SOURCE_DIR}/contrib/scalar/scalar.c) + target_link_libraries(scalar common-main) + set_target_properties(scalar PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_BINARY_DIR}/contrib/scalar) + set_target_properties(scalar PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_BINARY_DIR}/contrib/scalar) +endif() + parse_makefile_for_executables(git_builtin_extra "BUILT_INS") option(SKIP_DASHED_BUILT_INS "Skip hardlinking the dashed versions of the built-ins") @@ -1037,6 +1044,13 @@ string(REPLACE "@@BUILD_DIR@@" "${CMAKE_BINARY_DIR}" content "${content}") string(REPLACE "@@PROG@@" "git-cvsserver" content "${content}") file(WRITE ${CMAKE_BINARY_DIR}/bin-wrappers/git-cvsserver ${content}) +if(DEFINED ENV{INCLUDE_SCALAR} AND NOT ENV{INCLUDE_SCALAR} STREQUAL "") + file(STRINGS ${CMAKE_SOURCE_DIR}/wrap-for-bin.sh content NEWLINE_CONSUME) + string(REPLACE "@@BUILD_DIR@@" "${CMAKE_BINARY_DIR}" content "${content}") + string(REPLACE "@@PROG@@" "contrib/scalar/scalar${EXE_EXTENSION}" content "${content}") + file(WRITE ${CMAKE_BINARY_DIR}/bin-wrappers/scalar ${content}) +endif() + #options for configuring test options option(PERL_TESTS "Perform tests that use perl" ON) option(PYTHON_TESTS "Perform tests that use python" ON) From 3fbc77ef1aee3660af8a5563e5347b839085f03c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 18 May 2021 22:48:24 +0200 Subject: [PATCH 04/35] git_config_set_multivar_in_file_gently(): add a lock timeout In particular when multiple processes want to write to the config simultaneously, it would come in handy to not fail immediately when another process locked the config, but to gently try again. This will help with Scalar's functional test suite which wants to register multiple repositories for maintenance semi-simultaneously. As not all code paths calling this function read the config (e.g. `git config`), we have to read the config setting via `git_config_get_ulong()`. Signed-off-by: Johannes Schindelin --- Documentation/config/core.txt | 9 +++++++++ config.c | 8 +++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index d52fc67af13efc..2ae54f3df3b4d4 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -749,3 +749,12 @@ core.abbrev:: If set to "no", no abbreviation is made and the object names are shown in their full length. The minimum length is 4. + +core.configWriteLockTimeoutMS:: + When processes try to write to the config concurrently, it is likely + that one process "wins" and the other process(es) fail to lock the + config file. By configuring a timeout larger than zero, Git can be + told to try to lock the config again a couple times within the + specified timeout. If the timeout is configure to zero (which is the + default), Git will fail immediately when the config is already + locked. diff --git a/config.c b/config.c index 7176f800e1607b..513f1dcb15d322 100644 --- a/config.c +++ b/config.c @@ -3202,6 +3202,7 @@ int git_config_set_multivar_in_file_gently(const char *config_filename, const char *value_pattern, unsigned flags) { + static unsigned long timeout_ms = ULONG_MAX; int fd = -1, in_fd = -1; int ret; struct lock_file lock = LOCK_INIT; @@ -3222,11 +3223,16 @@ int git_config_set_multivar_in_file_gently(const char *config_filename, if (!config_filename) config_filename = filename_buf = git_pathdup("config"); + if ((long)timeout_ms < 0 && + git_config_get_ulong("core.configWriteLockTimeoutMS", &timeout_ms)) + timeout_ms = 0; + /* * The lock serves a purpose in addition to locking: the new * contents of .git/config will be written into it. */ - fd = hold_lock_file_for_update(&lock, config_filename, 0); + fd = hold_lock_file_for_update_timeout(&lock, config_filename, 0, + timeout_ms); if (fd < 0) { error_errno(_("could not lock config file %s"), config_filename); ret = CONFIG_NO_LOCK; From dc07149673b3efdb0660946be4d23776ce434692 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 19 Sep 2022 15:04:35 -0400 Subject: [PATCH 05/35] maintenance: make unregister idempotent The 'git maintenance unregister' subcommand has a step that removes the current repository from the multi-valued maitenance.repo config key. This fails if the repository is not listed in that key. This makes running 'git maintenance unregister' twice result in a failure in the second instance. Make this task idempotent, since the end result is the same in both cases: maintenance will no longer run on this repository. Signed-off-by: Derrick Stolee --- builtin/gc.c | 24 +++++++++++++++++++----- t/t7900-maintenance.sh | 5 ++++- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/builtin/gc.c b/builtin/gc.c index 7e3db00df37696..05a6f498615d40 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -1528,9 +1528,13 @@ static int maintenance_unregister(int argc, const char **argv, const char *prefi struct option options[] = { OPT_END(), }; - int rc; + const char *key = "maintenance.repo"; + int rc = 0; struct child_process config_unset = CHILD_PROCESS_INIT; char *maintpath = get_maintpath(); + int found = 0; + struct string_list_item *item; + const struct string_list *list = git_config_get_value_multi(key); argc = parse_options(argc, argv, prefix, options, builtin_maintenance_unregister_usage, 0); @@ -1538,11 +1542,21 @@ static int maintenance_unregister(int argc, const char **argv, const char *prefi usage_with_options(builtin_maintenance_unregister_usage, options); - config_unset.git_cmd = 1; - strvec_pushl(&config_unset.args, "config", "--global", "--unset", - "--fixed-value", "maintenance.repo", maintpath, NULL); + for_each_string_list_item(item, list) { + if (!strcmp(maintpath, item->string)) { + found = 1; + break; + } + } + + if (found) { + config_unset.git_cmd = 1; + strvec_pushl(&config_unset.args, "config", "--global", "--unset", + "--fixed-value", key, maintpath, NULL); + + rc = run_command(&config_unset); + } - rc = run_command(&config_unset); free(maintpath); return rc; } diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index 2724a44fe3ef24..3747e4a14f8f02 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -493,7 +493,10 @@ test_expect_success 'register and unregister' ' git maintenance unregister && git config --global --get-all maintenance.repo >actual && - test_cmp before actual + test_cmp before actual && + + # Expect unregister to be idempotent. + git maintenance unregister ' test_expect_success !MINGW 'register and unregister with regex metacharacters' ' From 54f277e3836e9d66ab3f8b34eceee69100863404 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 4 May 2021 10:32:24 +0200 Subject: [PATCH 06/35] git help: special-case `scalar` With this commit, `git help scalar` will open the appropriate manual or HTML page (instead of looking for `gitscalar`). Signed-off-by: Johannes Schindelin --- builtin/help.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/builtin/help.c b/builtin/help.c index 09ac4289f13065..6f2796f211e24c 100644 --- a/builtin/help.c +++ b/builtin/help.c @@ -440,6 +440,8 @@ static const char *cmd_to_page(const char *git_cmd) return git_cmd; else if (is_git_command(git_cmd)) return xstrfmt("git-%s", git_cmd); + else if (!strcmp("scalar", git_cmd)) + return xstrdup(git_cmd); else return xstrfmt("git%s", git_cmd); } From 0bf41b28e2682092e3db0d1a4b2405e7ded7fe08 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 14 Apr 2021 20:33:39 +0200 Subject: [PATCH 07/35] ci: also run the `scalar` tests Since Scalar depends on `libgit.a`, it makes sense to ensure in the CI and the PR builds that it does not get broken in case of industrious refactorings of the core Git code (speaking from experience here). Signed-off-by: Johannes Schindelin --- .github/workflows/main.yml | 15 +++++++++++++++ ci/run-build-and-tests.sh | 2 ++ ci/run-test-slice.sh | 8 ++++++-- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 92063286fffbaf..7dc5373ec5736c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -91,6 +91,13 @@ jobs: HOME: ${{runner.workspace}} NO_PERL: 1 run: . /etc/profile && ci/make-test-artifacts.sh artifacts + - name: build Scalar + shell: bash + run: | + make -C contrib/scalar && + mkdir -p artifacts/bin-wrappers artifacts/contrib/scalar && + cp contrib/scalar/scalar.exe artifacts/contrib/scalar/ && + cp bin-wrappers/scalar artifacts/bin-wrappers/ - name: zip up tracked files run: git archive -o artifacts/tracked.tar.gz HEAD - name: upload tracked files and build artifacts @@ -160,6 +167,8 @@ jobs: run: compat\vcbuild\vcpkg_copy_dlls.bat release ${{ matrix.arch }}-windows - name: generate Visual Studio solution shell: bash + env: + INCLUDE_SCALAR: YesPlease run: | cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/${{ matrix.arch }}-windows \ -DNO_GETTEXT=YesPlease -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON -DCMAKE_GENERATOR_PLATFORM=${{ matrix.arch }} -DVCPKG_ARCH=${{ matrix.arch }}-windows -DHOST_CPU=${{ matrix.arch }} @@ -173,6 +182,12 @@ jobs: run: | mkdir -p artifacts && eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts NO_GETTEXT=YesPlease 2>&1 | grep ^tar)" + - name: copy Scalar + shell: bash + run: | + mkdir -p artifacts/bin-wrappers artifacts/contrib/scalar && + cp contrib/scalar/scalar.exe artifacts/contrib/scalar/ && + cp bin-wrappers/scalar artifacts/bin-wrappers/ - name: zip up tracked files run: git archive -o artifacts/tracked.tar.gz HEAD - name: upload tracked files and build artifacts diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index f3195eb265f37b..7dd7c74e4734c1 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -52,4 +52,6 @@ case " $MAKE_TARGETS " in *" all "*) make -C contrib/subtree test;; esac +make -C contrib/scalar $MAKE_TARGETS + save_good_tree diff --git a/ci/run-test-slice.sh b/ci/run-test-slice.sh index 279aff4da9e4db..b9167d9803d932 100755 --- a/ci/run-test-slice.sh +++ b/ci/run-test-slice.sh @@ -10,7 +10,11 @@ group "Run tests" make --quiet -C t T="$(cd t && tr '\n' ' ')" || handle_failed_tests -# Run the git subtree tests only if main tests succeeded -test 0 != "$1" || make -C contrib/subtree test +if test 0 = "$1" +then + # Run the git subtree & scalar tests only if main tests succeeded + make -C contrib/subtree test && + make -C contrib/scalar test +fi check_unignored_build_artifacts From 8740613978de07a08f6d5189c108f621c91c06b2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 18 May 2021 23:22:56 +0200 Subject: [PATCH 08/35] scalar: set the config write-lock timeout to 150ms By default, Git fails immediately when locking a config file for writing fails due to an existing lock. With this change, Scalar-registered repositories will fall back to trying a couple times within a 150ms timeout. Signed-off-by: Johannes Schindelin --- contrib/scalar/scalar.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/contrib/scalar/scalar.c b/contrib/scalar/scalar.c index 642d16124eb20c..a91059c1ed983e 100644 --- a/contrib/scalar/scalar.c +++ b/contrib/scalar/scalar.c @@ -166,6 +166,7 @@ static int set_recommended_config(int reconfigure) { "core.autoCRLF", "false" }, { "core.safeCRLF", "false" }, { "fetch.showForcedUpdates", "false" }, + { "core.configWriteLockTimeoutMS", "150" }, { NULL, NULL }, }; int i; @@ -207,16 +208,25 @@ static int set_recommended_config(int reconfigure) static int toggle_maintenance(int enable) { + unsigned long ul; + + if (git_config_get_ulong("core.configWriteLockTimeoutMS", &ul)) + git_config_push_parameter("core.configWriteLockTimeoutMS=150"); + return run_git("maintenance", enable ? "start" : "unregister", NULL); } static int add_or_remove_enlistment(int add) { int res; + unsigned long ul; if (!the_repository->worktree) die(_("Scalar enlistments require a worktree")); + if (git_config_get_ulong("core.configWriteLockTimeoutMS", &ul)) + git_config_push_parameter("core.configWriteLockTimeoutMS=150"); + res = run_git("config", "--global", "--get", "--fixed-value", "scalar.repo", the_repository->worktree, NULL); From 6caab7b07f869bb4c820fc080cfc981b592905fa Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 16 Jun 2021 10:01:37 -0400 Subject: [PATCH 09/35] scalar: add docs from microsoft/scalar These docs have been altered to fit the version implemented in C within microsoft/git. This means in particular that the advanced.md file no longer applied at all. Some other areas were removed or significantly edited. Signed-off-by: Derrick Stolee --- contrib/scalar/docs/faq.md | 51 ++++++++++++++ contrib/scalar/docs/getting-started.md | 98 ++++++++++++++++++++++++++ contrib/scalar/docs/index.md | 50 +++++++++++++ contrib/scalar/docs/philosophy.md | 66 +++++++++++++++++ contrib/scalar/docs/troubleshooting.md | 20 ++++++ 5 files changed, 285 insertions(+) create mode 100644 contrib/scalar/docs/faq.md create mode 100644 contrib/scalar/docs/getting-started.md create mode 100644 contrib/scalar/docs/index.md create mode 100644 contrib/scalar/docs/philosophy.md create mode 100644 contrib/scalar/docs/troubleshooting.md diff --git a/contrib/scalar/docs/faq.md b/contrib/scalar/docs/faq.md new file mode 100644 index 00000000000000..a14f78a996d5d5 --- /dev/null +++ b/contrib/scalar/docs/faq.md @@ -0,0 +1,51 @@ +Frequently Asked Questions +========================== + +Using Scalar +------------ + +### I don't want a sparse clone, I want every file after I clone! + +Run `scalar clone --full-clone ` to initialize your repo to include +every file. You can switch to a sparse-checkout later by running +`git sparse-checkout init --cone`. + +### I already cloned without `--full-clone`. How do I get everything? + +Run `git sparse-checkout disable`. + +Scalar Design Decisions +----------------------- + +There may be many design decisions within Scalar that are confusing at first +glance. Some of them may cause friction when you use Scalar with your existing +repos and existing habits. + +> Scalar has the most benefit when users design repositories +> with efficient patterns. + +For example: Scalar uses the sparse-checkout feature to limit the size of the +working directory within a large monorepo. It is designed to work efficiently +with monorepos that are highly componentized, allowing most developers to +need many fewer files in their daily work. + +### Why does `scalar clone` create a `/src` folder? + +Scalar uses a file system watcher to keep track of changes under this `src` folder. +Any activity in this folder is assumed to be important to Git operations. By +creating the `src` folder, we are making it easy for your build system to +create output folders outside the `src` directory. We commonly see systems +create folders for build outputs and package downloads. Scalar itself creates +these folders during its builds. + +Your build system may create build artifacts such as `.obj` or `.lib` files +next to your source code. These are commonly "hidden" from Git using +`.gitignore` files. Having such artifacts in your source tree creates +additional work for Git because it needs to look at these files and match them +against the `.gitignore` patterns. + +By following the `src` pattern Scalar tries to establish and placing your build +intermediates and outputs parallel with the `src` folder and not inside it, +you can help optimize Git command performance for developers in the repository +by limiting the number of files Git needs to consider for many common +operations. diff --git a/contrib/scalar/docs/getting-started.md b/contrib/scalar/docs/getting-started.md new file mode 100644 index 00000000000000..dec60ce407713a --- /dev/null +++ b/contrib/scalar/docs/getting-started.md @@ -0,0 +1,98 @@ +Getting Started +=============== + +Registering existing Git repos +------------------------------ + +To add a repository to the list of registered repos, run `scalar register []`. +If `` is not provided, then the "current repository" is discovered from +the working directory by scanning the parent paths for a path containing a `.git` +folder, possibly inside a `src` folder. + +To see which repositories are currently tracked by the service, run +`scalar list`. + +Run `scalar unregister []` to remove the repo from this list. + +Creating a new Scalar clone +--------------------------------------------------- + +The `clone` verb creates a local enlistment of a remote repository using the +partial clone feature available e.g. on GitHub. + + +``` +scalar clone [options] [] +``` + +Create a local copy of the repository at ``. If specified, create the `` +directory and place the repository there. Otherwise, the last section of the `` +will be used for ``. + +At the end, the repo is located at `/src`. By default, the sparse-checkout +feature is enabled and the only files present are those in the root of your +Git repository. Use `git sparse-checkout set` to expand the set of directories +you want to see, or `git sparse-checkout disable` to expand to all files. You +can explore the subdirectories outside your sparse-checkout specification using +`git ls-tree HEAD`. + +### Sparse Repo Mode + +By default, Scalar reduces your working directory to only the files at the +root of the repository. You need to add the folders you care about to build up +to your working set. + +* `scalar clone ` + * Please choose the **Clone with HTTPS** option in the `Clone Repository` dialog in Azure Repos, not **Clone with SSH**. +* `cd \src` +* At this point, your `src` directory only contains files that appear in your root + tree. No folders are populated. +* Set the directory list for your sparse-checkout using: + 1. `git sparse-checkout set ...` + 2. `git sparse-checkout set --stdin < dir-list.txt` +* Run git commands as you normally would. +* To fully populate your working directory, run `git sparse-checkout disable`. + +If instead you want to start with all files on-disk, you can clone with the +`--full-clone` option. To enable sparse-checkout after the fact, run +`git sparse-checkout init --cone`. This will initialize your sparse-checkout +patterns to only match the files at root. + +If you are unfamiliar with what directories are available in the repository, +then you can run `git ls-tree -d --name-only HEAD` to discover the directories +at root, or `git ls-tree -d --name-only HEAD ` to discover the directories +in ``. + +### Options + +These options allow a user to customize their initial enlistment. + +* `--full-clone`: If specified, do not initialize the sparse-checkout feature. + All files will be present in your `src` directory. This uses a Git partial + clone: blobs are downloaded on demand. + +* `--branch=`: Specify the branch to checkout after clone. + +### Advanced Options + +The options below are not intended for use by a typical user. These are +usually used by build machines to create a temporary enlistment that +operates on a single commit. + +* `--single-branch`: Use this option to only download metadata for the branch + that will be checked out. This is helpful for build machines that target + a remote with many branches. Any `git fetch` commands after the clone will + still ask for all branches. + +* `--no-prefetch`: Use this option to not prefetch commits after clone. This + is not recommended for anyone planning to use their clone for history + traversal. Use of this option will make commands like `git log` or + `git pull` extremely slow and is therefore not recommended. + +Removing a Scalar Clone +----------------------- + +Since the `scalar clone` command sets up a file-system watcher (when available), +that watcher could prevent deleting the enlistment. Run `scalar delete ` +from outside of your enlistment to unregister the enlistment from the filesystem +watcher and delete the enlistment at ``. diff --git a/contrib/scalar/docs/index.md b/contrib/scalar/docs/index.md new file mode 100644 index 00000000000000..f9f5ab06e09253 --- /dev/null +++ b/contrib/scalar/docs/index.md @@ -0,0 +1,50 @@ +Scalar: Enabling Git at Scale +============================= + +Scalar is a tool that helps Git scale to some of the largest Git repositories. +It achieves this by enabling some advanced Git features, such as: + +* *Partial clone:* reduces time to get a working repository by not + downloading all Git objects right away. + +* *Background prefetch:* downloads Git object data from all remotes every + hour, reducing the amount of time for foreground `git fetch` calls. + +* *Sparse-checkout:* limits the size of your working directory. + +* *File system monitor:* tracks the recently modified files and eliminates + the need for Git to scan the entire worktree. + +* *Commit-graph:* accelerates commit walks and reachability calculations, + speeding up commands like `git log`. + +* *Multi-pack-index:* enables fast object lookups across many pack-files. + +* *Incremental repack:* Repacks the packed Git data into fewer pack-file + without disrupting concurrent commands by using the multi-pack-index. + +By running `scalar register` in any Git repo, Scalar will automatically enable +these features for that repo (except partial clone) and start running suggested +maintenance in the background using +[the `git maintenance` feature](https://git-scm.com/docs/git-maintenance). + +Repos cloned with the `scalar clone` command use partial clone to significantly +reduce the amount of data required to get started using a repository. By +delaying all blob downloads until they are required, Scalar allows you to work +with very large repositories quickly. + +Documentation +------------- + +* [Getting Started](getting-started.md): Get started with Scalar. + Includes `scalar register`, `scalar unregister`, `scalar clone`, and + `scalar delete`. + +* [Troubleshooting](troubleshooting.md): + Collect diagnostic information or update custom settings. Includes + `scalar diagnose`. + +* [The Philosophy of Scalar](philosophy.md): Why does Scalar work the way + it does, and how do we make decisions about its future? + +* [Frequently Asked Questions](faq.md) diff --git a/contrib/scalar/docs/philosophy.md b/contrib/scalar/docs/philosophy.md new file mode 100644 index 00000000000000..51486a75e41f0d --- /dev/null +++ b/contrib/scalar/docs/philosophy.md @@ -0,0 +1,66 @@ +The Philosophy of Scalar +======================== + +The team building Scalar has **opinions** about Git performance. Scalar +takes out the guesswork by automatically configuring your Git repositories +to take advantage of the latest and greatest features. It is difficult to +say that these are the absolute best settings for every repository, but +these settings do work for some of the largest repositories in the world. + +Scalar intends to do very little more than the standard Git client. We +actively implement new features into Git instead of Scalar, then update +Scalar only to configure those new settings. In particular, we ported +features like background maintenance to Git to make Scalar simpler and +make Git more powerful. + +Services such as GitHub support partial clone , a standard adopted by the Git +project to download only part of the Git objects when cloning, and fetching +further objects on demand. If your hosting service supports partial clone, then +we absolutely recommend it as a way to greatly speed up your clone and fetch +times and to reduce how much disk space your Git repository requires. Scalar +will help with this! + +Most of the value of Scalar can be found in the core Git client. However, most +of the advanced features that really optimize Git's performance are off by +default for compatibility reasons. To really take advantage of Git's latest and +greatest features, you either need to study the [`git config` +documentation](https://git-scm.com/docs/git-config) and regularly read [the Git +release notes](https://github.com/git/git/tree/master/Documentation/RelNotes). +Even if you do all that work and customize your Git settings on your machines, +you likely will want to share those settings with other team members. Or, you +can just use Scalar! + +Using `scalar register` on an existing Git repository will give you these +benefits: + +* Additional compression of your `.git/index` file. +* Hourly background `git fetch` operations, keeping you in-sync with your + remotes. +* Advanced data structures, such as the `commit-graph` and `multi-pack-index` + are updated automatically in the background. +* If using macOS or Windows, then Scalar configures Git's builtin File System + Monitor, providing faster commands such as `git status` or `git add`. + +Additionally, if you use `scalar clone` to create a new repository, then +you will automatically get these benefits: + +* Use Git's partial clone feature to only download the files you need for + your current checkout. +* Use Git's [sparse-checkout feature][sparse-checkout] to minimize the + number of files required in your working directory. + [Read more about sparse-checkout here.][sparse-checkout-blog] +* Create the Git repository inside `/src` to make it easy to + place build artifacts outside of the Git repository, such as in + `/bin` or `/packages`. + +We also admit that these **opinions** can always be improved! If you have +an idea of how to improve our setup, consider +[creating an issue](https://github.com/microsoft/scalar/issues/new) or +contributing a pull request! Some [existing](https://github.com/microsoft/scalar/issues/382) +[issues](https://github.com/microsoft/scalar/issues/388) have already +improved our configuration settings and roadmap! + +[gvfs-protocol]: https://github.com/microsoft/VFSForGit/blob/HEAD/Protocol.md +[microsoft-git]: https://github.com/microsoft/git +[sparse-checkout]: https://git-scm.com/docs/git-sparse-checkout +[sparse-checkout-blog]: https://github.blog/2020-01-17-bring-your-monorepo-down-to-size-with-sparse-checkout/ diff --git a/contrib/scalar/docs/troubleshooting.md b/contrib/scalar/docs/troubleshooting.md new file mode 100644 index 00000000000000..8ec56ad437ff09 --- /dev/null +++ b/contrib/scalar/docs/troubleshooting.md @@ -0,0 +1,20 @@ +Troubleshooting +=============== + +Diagnosing Issues +----------------- + +The `scalar diagnose` command collects logs and config details for the current +repository. The resulting zip file helps root-cause issues. + +When run inside your repository, creates a zip file containing several important +files for that repository. This includes: + +* Configuration files from your `.git` folder, such as the `config` file, + `index`, `hooks`, and `refs`. + +* A summary of your Git object database, including the number of loose objects + and the names and sizes of pack-files. + +As the `diagnose` command completes, it provides the path of the resulting +zip file. This zip can be attached to bug reports to make the analysis easier. From c69c30c8e895a26b55898b14987b9ae900069ce8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 18 May 2021 22:32:58 +0200 Subject: [PATCH 10/35] scalar: implement the `help` subcommand It is merely handing off to `git help scalar`. Signed-off-by: Johannes Schindelin --- contrib/scalar/scalar.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/contrib/scalar/scalar.c b/contrib/scalar/scalar.c index a91059c1ed983e..70a376cd7da70b 100644 --- a/contrib/scalar/scalar.c +++ b/contrib/scalar/scalar.c @@ -829,6 +829,25 @@ static int cmd_delete(int argc, const char **argv) return res; } +static int cmd_help(int argc, const char **argv) +{ + struct option options[] = { + OPT_END(), + }; + const char * const usage[] = { + N_("scalar help"), + NULL + }; + + argc = parse_options(argc, argv, NULL, options, + usage, 0); + + if (argc != 0) + usage_with_options(usage, options); + + return run_git("help", "scalar", NULL); +} + static int cmd_version(int argc, const char **argv) { int verbose = 0, build_options = 0; @@ -868,6 +887,7 @@ static struct { { "run", cmd_run }, { "reconfigure", cmd_reconfigure }, { "delete", cmd_delete }, + { "help", cmd_help }, { "version", cmd_version }, { "diagnose", cmd_diagnose }, { NULL, NULL}, From 22b7e73e0e86daec4be320600c9ee4915a141db8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 16 Nov 2021 18:10:47 +0100 Subject: [PATCH 11/35] scalar: move it out of contrib/ With this patch, `scalar` becomes a fully-supported top-level Git command. The Scalar executable is installed by default as well as its documentation, and its (minimal) regression test is run as part of the full test suite. Signed-off-by: Johannes Schindelin --- .github/workflows/main.yml | 15 ---- .gitignore | 1 + Documentation/Makefile | 2 + {contrib/scalar => Documentation}/scalar.txt | 4 +- .../docs => Documentation/scalar}/faq.md | 0 .../scalar}/getting-started.md | 0 .../docs => Documentation/scalar}/index.md | 0 .../scalar}/philosophy.md | 0 .../scalar}/troubleshooting.md | 0 Makefile | 17 ++-- ci/run-build-and-tests.sh | 2 - ci/run-test-slice.sh | 5 +- contrib/buildsystems/CMakeLists.txt | 19 +---- contrib/scalar/.gitignore | 5 -- contrib/scalar/Makefile | 69 ---------------- contrib/scalar/t/Makefile | 81 ------------------- contrib/scalar/scalar.c => scalar.c | 0 {contrib/scalar/t => t}/t9099-scalar.sh | 8 +- 18 files changed, 21 insertions(+), 207 deletions(-) rename {contrib/scalar => Documentation}/scalar.txt (99%) rename {contrib/scalar/docs => Documentation/scalar}/faq.md (100%) rename {contrib/scalar/docs => Documentation/scalar}/getting-started.md (100%) rename {contrib/scalar/docs => Documentation/scalar}/index.md (100%) rename {contrib/scalar/docs => Documentation/scalar}/philosophy.md (100%) rename {contrib/scalar/docs => Documentation/scalar}/troubleshooting.md (100%) delete mode 100644 contrib/scalar/.gitignore delete mode 100644 contrib/scalar/Makefile delete mode 100644 contrib/scalar/t/Makefile rename contrib/scalar/scalar.c => scalar.c (100%) rename {contrib/scalar/t => t}/t9099-scalar.sh (97%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7dc5373ec5736c..92063286fffbaf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -91,13 +91,6 @@ jobs: HOME: ${{runner.workspace}} NO_PERL: 1 run: . /etc/profile && ci/make-test-artifacts.sh artifacts - - name: build Scalar - shell: bash - run: | - make -C contrib/scalar && - mkdir -p artifacts/bin-wrappers artifacts/contrib/scalar && - cp contrib/scalar/scalar.exe artifacts/contrib/scalar/ && - cp bin-wrappers/scalar artifacts/bin-wrappers/ - name: zip up tracked files run: git archive -o artifacts/tracked.tar.gz HEAD - name: upload tracked files and build artifacts @@ -167,8 +160,6 @@ jobs: run: compat\vcbuild\vcpkg_copy_dlls.bat release ${{ matrix.arch }}-windows - name: generate Visual Studio solution shell: bash - env: - INCLUDE_SCALAR: YesPlease run: | cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/${{ matrix.arch }}-windows \ -DNO_GETTEXT=YesPlease -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON -DCMAKE_GENERATOR_PLATFORM=${{ matrix.arch }} -DVCPKG_ARCH=${{ matrix.arch }}-windows -DHOST_CPU=${{ matrix.arch }} @@ -182,12 +173,6 @@ jobs: run: | mkdir -p artifacts && eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts NO_GETTEXT=YesPlease 2>&1 | grep ^tar)" - - name: copy Scalar - shell: bash - run: | - mkdir -p artifacts/bin-wrappers artifacts/contrib/scalar && - cp contrib/scalar/scalar.exe artifacts/contrib/scalar/ && - cp bin-wrappers/scalar artifacts/bin-wrappers/ - name: zip up tracked files run: git archive -o artifacts/tracked.tar.gz HEAD - name: upload tracked files and build artifacts diff --git a/.gitignore b/.gitignore index 265c5a772cf949..a867f6113c1057 100644 --- a/.gitignore +++ b/.gitignore @@ -194,6 +194,7 @@ /config-list.h /command-list.h /hook-list.h +/scalar *.tar.gz *.dsc *.deb diff --git a/Documentation/Makefile b/Documentation/Makefile index 849af6da307b41..ad2f3dce790169 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -22,6 +22,8 @@ MAN1_TXT += git.txt MAN1_TXT += gitk.txt MAN1_TXT += gitweb.txt +MAN1_TXT += scalar.txt + # man5 / man7 guides (note: new guides should also be added to command-list.txt) MAN5_TXT += gitattributes.txt MAN5_TXT += gitformat-bundle.txt diff --git a/contrib/scalar/scalar.txt b/Documentation/scalar.txt similarity index 99% rename from contrib/scalar/scalar.txt rename to Documentation/scalar.txt index 1a12dc450774c2..f33436c7f65ff9 100644 --- a/contrib/scalar/scalar.txt +++ b/Documentation/scalar.txt @@ -161,6 +161,6 @@ SEE ALSO -------- linkgit:git-clone[1], linkgit:git-maintenance[1]. -Scalar +GIT --- -Associated with the linkgit:git[1] suite +Part of the linkgit:git[1] suite diff --git a/contrib/scalar/docs/faq.md b/Documentation/scalar/faq.md similarity index 100% rename from contrib/scalar/docs/faq.md rename to Documentation/scalar/faq.md diff --git a/contrib/scalar/docs/getting-started.md b/Documentation/scalar/getting-started.md similarity index 100% rename from contrib/scalar/docs/getting-started.md rename to Documentation/scalar/getting-started.md diff --git a/contrib/scalar/docs/index.md b/Documentation/scalar/index.md similarity index 100% rename from contrib/scalar/docs/index.md rename to Documentation/scalar/index.md diff --git a/contrib/scalar/docs/philosophy.md b/Documentation/scalar/philosophy.md similarity index 100% rename from contrib/scalar/docs/philosophy.md rename to Documentation/scalar/philosophy.md diff --git a/contrib/scalar/docs/troubleshooting.md b/Documentation/scalar/troubleshooting.md similarity index 100% rename from contrib/scalar/docs/troubleshooting.md rename to Documentation/scalar/troubleshooting.md diff --git a/Makefile b/Makefile index 0370c1d4432e79..23394f4710754d 100644 --- a/Makefile +++ b/Makefile @@ -697,6 +697,11 @@ all:: $(FUZZ_OBJS) FUZZ_PROGRAMS += $(patsubst %.o,%,$(FUZZ_OBJS)) +SCALAR_OBJS := scalar.o + +PROGRAMS += scalar$(X) +BINDIR_PROGRAMS_NEED_X += scalar + # Empty... EXTRA_PROGRAMS = @@ -2583,6 +2588,7 @@ OBJECTS += $(GIT_OBJS) OBJECTS += $(PROGRAM_OBJS) OBJECTS += $(TEST_OBJS) OBJECTS += $(XDIFF_OBJS) +OBJECTS += $(SCALAR_OBJS) OBJECTS += $(FUZZ_OBJS) OBJECTS += $(REFTABLE_OBJS) $(REFTABLE_TEST_OBJS) @@ -2590,10 +2596,6 @@ ifndef NO_CURL OBJECTS += http.o http-walker.o remote-curl.o endif -SCALAR_SOURCES := contrib/scalar/scalar.c -SCALAR_OBJECTS := $(SCALAR_SOURCES:c=o) -OBJECTS += $(SCALAR_OBJECTS) - .PHONY: objects objects: $(OBJECTS) @@ -2726,9 +2728,8 @@ $(REMOTE_CURL_PRIMARY): remote-curl.o http.o http-walker.o GIT-LDFLAGS $(GITLIBS $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \ $(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS) -contrib/scalar/scalar$X: $(SCALAR_OBJECTS) GIT-LDFLAGS $(GITLIBS) - $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) \ - $(filter %.o,$^) $(LIBS) +scalar$X: $(SCALAR_OBJS) GIT-LDFLAGS $(GITLIBS) + $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(LIBS) $(LIB_FILE): $(LIB_OBJS) $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ @@ -3097,7 +3098,7 @@ bin-wrappers/%: wrap-for-bin.sh $(call mkdir_p_parent_template) $(QUIET_GEN)sed -e '1s|#!.*/sh|#!$(SHELL_PATH_SQ)|' \ -e 's|@@BUILD_DIR@@|$(shell pwd)|' \ - -e 's|@@PROG@@|$(patsubst test-%,t/helper/test-%$(X),$(@F))$(patsubst git%,$(X),$(filter $(@F),$(BINDIR_PROGRAMS_NEED_X)))|' < $< > $@ && \ + -e 's|@@PROG@@|$(patsubst test-%,t/helper/test-%$(X),$(@F))$(patsubst scalar,$(X),$(patsubst git%,$(X),$(filter $(@F),$(BINDIR_PROGRAMS_NEED_X))))|' < $< > $@ && \ chmod +x $@ # GNU make supports exporting all variables by "export" without parameters. diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index 7dd7c74e4734c1..f3195eb265f37b 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -52,6 +52,4 @@ case " $MAKE_TARGETS " in *" all "*) make -C contrib/subtree test;; esac -make -C contrib/scalar $MAKE_TARGETS - save_good_tree diff --git a/ci/run-test-slice.sh b/ci/run-test-slice.sh index b9167d9803d932..03667da5e5f569 100755 --- a/ci/run-test-slice.sh +++ b/ci/run-test-slice.sh @@ -12,9 +12,8 @@ handle_failed_tests if test 0 = "$1" then - # Run the git subtree & scalar tests only if main tests succeeded - make -C contrib/subtree test && - make -C contrib/scalar test + # Run the git subtree tests only if main tests succeeded + make -C contrib/subtree test fi check_unignored_build_artifacts diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index d1b0fdf94a3348..718ad7aa28ef92 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -792,6 +792,9 @@ target_link_libraries(git-sh-i18n--envsubst common-main) add_executable(git-shell ${CMAKE_SOURCE_DIR}/shell.c) target_link_libraries(git-shell common-main) +add_executable(scalar ${CMAKE_SOURCE_DIR}/scalar.c) +target_link_libraries(scalar common-main) + if(CURL_FOUND) add_library(http_obj OBJECT ${CMAKE_SOURCE_DIR}/http.c) @@ -810,13 +813,6 @@ if(CURL_FOUND) endif() endif() -if(DEFINED ENV{INCLUDE_SCALAR} AND NOT ENV{INCLUDE_SCALAR} STREQUAL "") - add_executable(scalar ${CMAKE_SOURCE_DIR}/contrib/scalar/scalar.c) - target_link_libraries(scalar common-main) - set_target_properties(scalar PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_BINARY_DIR}/contrib/scalar) - set_target_properties(scalar PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_BINARY_DIR}/contrib/scalar) -endif() - parse_makefile_for_executables(git_builtin_extra "BUILT_INS") option(SKIP_DASHED_BUILT_INS "Skip hardlinking the dashed versions of the built-ins") @@ -1019,7 +1015,7 @@ endif() #wrapper scripts set(wrapper_scripts - git git-upload-pack git-receive-pack git-upload-archive git-shell git-remote-ext) + git git-upload-pack git-receive-pack git-upload-archive git-shell git-remote-ext scalar) set(wrapper_test_scripts test-fake-ssh test-tool) @@ -1044,13 +1040,6 @@ string(REPLACE "@@BUILD_DIR@@" "${CMAKE_BINARY_DIR}" content "${content}") string(REPLACE "@@PROG@@" "git-cvsserver" content "${content}") file(WRITE ${CMAKE_BINARY_DIR}/bin-wrappers/git-cvsserver ${content}) -if(DEFINED ENV{INCLUDE_SCALAR} AND NOT ENV{INCLUDE_SCALAR} STREQUAL "") - file(STRINGS ${CMAKE_SOURCE_DIR}/wrap-for-bin.sh content NEWLINE_CONSUME) - string(REPLACE "@@BUILD_DIR@@" "${CMAKE_BINARY_DIR}" content "${content}") - string(REPLACE "@@PROG@@" "contrib/scalar/scalar${EXE_EXTENSION}" content "${content}") - file(WRITE ${CMAKE_BINARY_DIR}/bin-wrappers/scalar ${content}) -endif() - #options for configuring test options option(PERL_TESTS "Perform tests that use perl" ON) option(PYTHON_TESTS "Perform tests that use python" ON) diff --git a/contrib/scalar/.gitignore b/contrib/scalar/.gitignore deleted file mode 100644 index 00441073f59cf5..00000000000000 --- a/contrib/scalar/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -/*.xml -/*.1 -/*.html -/*.exe -/scalar diff --git a/contrib/scalar/Makefile b/contrib/scalar/Makefile deleted file mode 100644 index df5ef37af94a82..00000000000000 --- a/contrib/scalar/Makefile +++ /dev/null @@ -1,69 +0,0 @@ -# The default target of this Makefile is... -all:: - -# Import tree-wide shared Makefile behavior and libraries -include ../../shared.mak - -include ../../config.mak.uname --include ../../config.mak.autogen --include ../../config.mak - -TARGETS = scalar$(X) scalar.o -GITLIBS = ../../common-main.o ../../libgit.a ../../xdiff/lib.a - -all:: scalar$(X) ../../bin-wrappers/scalar - -$(GITLIBS): - $(QUIET_SUBDIR0)../.. $(QUIET_SUBDIR1) $(subst ../../,,$@) - -$(TARGETS): $(GITLIBS) scalar.c - $(QUIET_SUBDIR0)../.. $(QUIET_SUBDIR1) $(patsubst %,contrib/scalar/%,$@) - -clean: - $(RM) $(TARGETS) ../../bin-wrappers/scalar - $(RM) scalar.1 scalar.html scalar.xml - -../../bin-wrappers/scalar: ../../wrap-for-bin.sh Makefile - @mkdir -p ../../bin-wrappers - $(QUIET_GEN)sed -e '1s|#!.*/sh|#!$(SHELL_PATH_SQ)|' \ - -e 's|@@BUILD_DIR@@|$(shell cd ../.. && pwd)|' \ - -e 's|@@PROG@@|contrib/scalar/scalar$(X)|' < $< > $@ && \ - chmod +x $@ - -test: all - $(MAKE) -C t - -INSTALL = install -prefix = $(HOME) -bindir = $(prefix)/bin -mandir ?= $(prefix)/share/man -man1dir = $(mandir)/man1 -htmldir ?= $(prefix)/share/doc/git-doc -DESTDIR_SQ = $(subst ','\'',$(DESTDIR)) -bindir_SQ = $(subst ','\'',$(bindir)) -man1dir_SQ = $(subst ','\'',$(man1dir)) -htmldir_SQ = $(subst ','\'',$(htmldir)) - -install: - $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(bindir_SQ)' - $(INSTALL) scalar$(X) '$(DESTDIR_SQ)$(bindir_SQ)' - -docs: scalar.html scalar.1 - -scalar.html: | scalar.1 # prevent them from trying to build `doc.dep` in parallel - -scalar.html scalar.1: scalar.txt - $(QUIET_SUBDIR0)../../Documentation$(QUIET_SUBDIR1) \ - MAN_TXT=../contrib/scalar/scalar.txt \ - ../contrib/scalar/$@ - $(QUIET)test scalar.1 != "$@" || mv ../../Documentation/$@ . - -install-doc: scalar.1 - $(INSTALL) -d -m 755 $(DESTDIR_SQ)$(man1dir_SQ) - $(INSTALL) -m 644 $^ $(DESTDIR_SQ)$(man1dir_SQ) - -install-html: scalar.html - $(INSTALL) -d -m 755 $(DESTDIR_SQ)$(htmldir_SQ) - $(INSTALL) -m 644 $^ $(DESTDIR_SQ)$(htmldir_SQ) - -.PHONY: $(GITLIBS) all clean test docs install install-doc install-html FORCE diff --git a/contrib/scalar/t/Makefile b/contrib/scalar/t/Makefile deleted file mode 100644 index 1ed174a8cf38e3..00000000000000 --- a/contrib/scalar/t/Makefile +++ /dev/null @@ -1,81 +0,0 @@ -# Import tree-wide shared Makefile behavior and libraries -include ../../../shared.mak - -# Run scalar tests -# -# Copyright (c) 2005,2021 Junio C Hamano, Johannes Schindelin -# - --include ../../../config.mak.autogen --include ../../../config.mak - -SHELL_PATH ?= $(SHELL) -PERL_PATH ?= /usr/bin/perl -RM ?= rm -f -PROVE ?= prove -DEFAULT_TEST_TARGET ?= test -TEST_LINT ?= test-lint - -ifdef TEST_OUTPUT_DIRECTORY -TEST_RESULTS_DIRECTORY = $(TEST_OUTPUT_DIRECTORY)/test-results -else -TEST_RESULTS_DIRECTORY = ../../../t/test-results -endif - -# Shell quote; -SHELL_PATH_SQ = $(subst ','\'',$(SHELL_PATH)) -PERL_PATH_SQ = $(subst ','\'',$(PERL_PATH)) -TEST_RESULTS_DIRECTORY_SQ = $(subst ','\'',$(TEST_RESULTS_DIRECTORY)) - -T = $(sort $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)) - -all: $(DEFAULT_TEST_TARGET) - -test: $(TEST_LINT) - $(MAKE) aggregate-results-and-cleanup - -prove: $(TEST_LINT) - @echo "*** prove ***"; GIT_CONFIG=.git/config $(PROVE) --exec '$(SHELL_PATH_SQ)' $(GIT_PROVE_OPTS) $(T) :: $(GIT_TEST_OPTS) - $(MAKE) clean-except-prove-cache - -$(T): - @echo "*** $@ ***"; GIT_CONFIG=.git/config '$(SHELL_PATH_SQ)' $@ $(GIT_TEST_OPTS) - -clean-except-prove-cache: - $(RM) -r 'trash directory'.* - $(RM) -r valgrind/bin - -clean: clean-except-prove-cache - $(RM) .prove - -test-lint: test-lint-duplicates test-lint-executable test-lint-shell-syntax - -test-lint-duplicates: - @dups=`echo $(T) | tr ' ' '\n' | sed 's/-.*//' | sort | uniq -d` && \ - test -z "$$dups" || { \ - echo >&2 "duplicate test numbers:" $$dups; exit 1; } - -test-lint-executable: - @bad=`for i in $(T); do test -x "$$i" || echo $$i; done` && \ - test -z "$$bad" || { \ - echo >&2 "non-executable tests:" $$bad; exit 1; } - -test-lint-shell-syntax: - @'$(PERL_PATH_SQ)' ../../../t/check-non-portable-shell.pl $(T) - -aggregate-results-and-cleanup: $(T) - $(MAKE) aggregate-results - $(MAKE) clean - -aggregate-results: - for f in '$(TEST_RESULTS_DIRECTORY_SQ)'/t*-*.counts; do \ - echo "$$f"; \ - done | '$(SHELL_PATH_SQ)' ../../../t/aggregate-results.sh - -valgrind: - $(MAKE) GIT_TEST_OPTS="$(GIT_TEST_OPTS) --valgrind" - -test-results: - mkdir -p test-results - -.PHONY: $(T) aggregate-results clean valgrind diff --git a/contrib/scalar/scalar.c b/scalar.c similarity index 100% rename from contrib/scalar/scalar.c rename to scalar.c diff --git a/contrib/scalar/t/t9099-scalar.sh b/t/t9099-scalar.sh similarity index 97% rename from contrib/scalar/t/t9099-scalar.sh rename to t/t9099-scalar.sh index dfb949f52eed04..62b92d361e2bfd 100755 --- a/contrib/scalar/t/t9099-scalar.sh +++ b/t/t9099-scalar.sh @@ -2,13 +2,7 @@ test_description='test the `scalar` command' -TEST_DIRECTORY=$PWD/../../../t -export TEST_DIRECTORY - -# Make it work with --no-bin-wrappers -PATH=$PWD/..:$PATH - -. ../../../t/test-lib.sh +. ./test-lib.sh GIT_TEST_MAINT_SCHEDULER="crontab:test-tool crontab ../cron.txt,launchctl:true,schtasks:true" export GIT_TEST_MAINT_SCHEDULER From eca27629715137bebafaf92bdf6f9a25b4df8561 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 May 2022 13:43:05 +0200 Subject: [PATCH 12/35] scalar (Windows): use forward slashes as directory separators Git traditionally uses those, not backslashes, ever. Signed-off-by: Johannes Schindelin --- scalar.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scalar.c b/scalar.c index 70a376cd7da70b..00682d30018ad0 100644 --- a/scalar.c +++ b/scalar.c @@ -41,6 +41,9 @@ static void setup_enlistment_directory(int argc, const char **argv, die(_("need a working directory")); strbuf_trim_trailing_dir_sep(&path); +#ifdef GIT_WINDOWS_NATIVE + convert_slashes(path.buf); +#endif /* check if currently in enlistment root with src/ workdir */ len = path.len; From 879236eb6f73d0f27b6e0a069b802f369753ffdc Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 17 Jun 2021 11:40:09 -0400 Subject: [PATCH 13/35] scalar: add retry logic to run_git() Use a fixed 3 tries total to see how that increases our chances of success for subcommands such as 'git fetch'. Signed-off-by: Derrick Stolee --- scalar.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scalar.c b/scalar.c index 00682d30018ad0..ba066f2e53dc04 100644 --- a/scalar.c +++ b/scalar.c @@ -70,12 +70,14 @@ static void setup_enlistment_directory(int argc, const char **argv, strbuf_release(&path); } +static int git_retries = 3; + static int run_git(const char *arg, ...) { struct strvec argv = STRVEC_INIT; va_list args; const char *p; - int res; + int res, attempts; va_start(args, arg); strvec_push(&argv, arg); @@ -83,7 +85,10 @@ static int run_git(const char *arg, ...) strvec_push(&argv, p); va_end(args); - res = run_command_v_opt(argv.v, RUN_GIT_CMD); + for (attempts = 0, res = 1; + res && attempts < git_retries; + attempts++) + res = run_command_v_opt(argv.v, RUN_GIT_CMD); strvec_clear(&argv); return res; From 0e84c352360fef9ade2f5635334b4316db8fd7cd Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 27 May 2021 07:26:11 +0200 Subject: [PATCH 14/35] scalar: support the `config` command for backwards compatibility The .NET version supported running `scalar config` to reconfigure the current enlistment, and now the C port does, too. Signed-off-by: Johannes Schindelin --- scalar.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scalar.c b/scalar.c index ba066f2e53dc04..cba905b36d1098 100644 --- a/scalar.c +++ b/scalar.c @@ -929,6 +929,9 @@ int cmd_main(int argc, const char **argv) argv++; argc--; + if (!strcmp(argv[0], "config")) + argv[0] = "reconfigure"; + for (i = 0; builtins[i].name; i++) if (!strcmp(builtins[i].name, argv[0])) return !!builtins[i].fn(argc, argv); From db89582dc7bbeefcf92d076cc7de606e810a9472 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 26 Apr 2021 22:08:18 +0200 Subject: [PATCH 15/35] scalar: implement a minimal JSON parser No grown-up C project comes without their own JSON parser. Just kidding! We need to parse a JSON result when determining which cache server to use. It would appear that searching for needles `"CacheServers":[`, `"Url":"` and `"GlobalDefault":true` _happens_ to work right now, it is fragile as it depends on no whitespace padding and on the order of the fields remaining as-is. Let's implement a super simple JSON parser (at the cost of being slightly inefficient) for that purpose. To avoid allocating a ton of memory, we implement a callback-based one. And to save on complexity, let's not even bother validating the input properly (we will just go ahead and instead rely on Azure Repos to produce correct JSON). Note: An alternative would have been to use existing solutions such as JSON-C, CentiJSON or JSMN. However, they are all a lot larger than the current solution; The smallest, JSMN, which does not even provide parsed string values (something we actually need) weighs in with 471 lines, while we get away with 182 + 29 lines for the C and the header file, respectively. Signed-off-by: Johannes Schindelin --- Makefile | 2 +- contrib/buildsystems/CMakeLists.txt | 2 +- json-parser.c | 182 ++++++++++++++++++++++++++++ json-parser.h | 29 +++++ 4 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 json-parser.c create mode 100644 json-parser.h diff --git a/Makefile b/Makefile index 7c969dcce67d32..4f66cc177c9633 100644 --- a/Makefile +++ b/Makefile @@ -697,7 +697,7 @@ all:: $(FUZZ_OBJS) FUZZ_PROGRAMS += $(patsubst %.o,%,$(FUZZ_OBJS)) -SCALAR_OBJS := scalar.o +SCALAR_OBJS := scalar.o json-parser.o PROGRAMS += scalar$(X) BINDIR_PROGRAMS_NEED_X += scalar diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index f97e31dd49dce6..c93606b94e549f 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -792,7 +792,7 @@ target_link_libraries(git-sh-i18n--envsubst common-main) add_executable(git-shell ${CMAKE_SOURCE_DIR}/shell.c) target_link_libraries(git-shell common-main) -add_executable(scalar ${CMAKE_SOURCE_DIR}/scalar.c) +add_executable(scalar ${CMAKE_SOURCE_DIR}/scalar.c ${CMAKE_SOURCE_DIR}/json-parser.c) target_link_libraries(scalar common-main) if(CURL_FOUND) diff --git a/json-parser.c b/json-parser.c new file mode 100644 index 00000000000000..30799e17dc0a04 --- /dev/null +++ b/json-parser.c @@ -0,0 +1,182 @@ +#include "cache.h" +#include "json-parser.h" + +static int reset_iterator(struct json_iterator *it) +{ + it->p = it->begin = it->json; + strbuf_release(&it->key); + strbuf_release(&it->string_value); + it->type = JSON_NULL; + return -1; +} + +static int parse_json_string(struct json_iterator *it, struct strbuf *out) +{ + const char *begin = it->p; + + if (*(it->p)++ != '"') + return error("expected double quote: '%.*s'", 5, begin), + reset_iterator(it); + + strbuf_reset(&it->string_value); +#define APPEND(c) strbuf_addch(out, c) + while (*it->p != '"') { + switch (*it->p) { + case '\0': + return error("incomplete string: '%s'", begin), + reset_iterator(it); + case '\\': + it->p++; + if (*it->p == '\\' || *it->p == '"') + APPEND(*it->p); + else if (*it->p == 'b') + APPEND(8); + else if (*it->p == 't') + APPEND(9); + else if (*it->p == 'n') + APPEND(10); + else if (*it->p == 'f') + APPEND(12); + else if (*it->p == 'r') + APPEND(13); + else if (*it->p == 'u') { + unsigned char binary[2]; + int i; + + if (hex_to_bytes(binary, it->p + 1, 2) < 0) + return error("invalid: '%.*s'", + 6, it->p - 1), + reset_iterator(it); + it->p += 4; + + i = (binary[0] << 8) | binary[1]; + if (i < 0x80) + APPEND(i); + else if (i < 0x0800) { + APPEND(0xc0 | ((i >> 6) & 0x1f)); + APPEND(0x80 | (i & 0x3f)); + } else if (i < 0x10000) { + APPEND(0xe0 | ((i >> 12) & 0x0f)); + APPEND(0x80 | ((i >> 6) & 0x3f)); + APPEND(0x80 | (i & 0x3f)); + } else { + APPEND(0xf0 | ((i >> 18) & 0x07)); + APPEND(0x80 | ((i >> 12) & 0x3f)); + APPEND(0x80 | ((i >> 6) & 0x3f)); + APPEND(0x80 | (i & 0x3f)); + } + } + break; + default: + APPEND(*it->p); + } + it->p++; + } + + it->end = it->p++; + return 0; +} + +static void skip_whitespace(struct json_iterator *it) +{ + while (isspace(*it->p)) + it->p++; +} + +int iterate_json(struct json_iterator *it) +{ + skip_whitespace(it); + it->begin = it->p; + + switch (*it->p) { + case '\0': + return reset_iterator(it), 0; + case 'n': + if (!starts_with(it->p, "null")) + return error("unexpected value: %.*s", 4, it->p), + reset_iterator(it); + it->type = JSON_NULL; + it->end = it->p = it->begin + 4; + break; + case 't': + if (!starts_with(it->p, "true")) + return error("unexpected value: %.*s", 4, it->p), + reset_iterator(it); + it->type = JSON_TRUE; + it->end = it->p = it->begin + 4; + break; + case 'f': + if (!starts_with(it->p, "false")) + return error("unexpected value: %.*s", 5, it->p), + reset_iterator(it); + it->type = JSON_FALSE; + it->end = it->p = it->begin + 5; + break; + case '-': case '.': + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + it->type = JSON_NUMBER; + it->end = it->p = it->begin + strspn(it->p, "-.0123456789"); + break; + case '"': + it->type = JSON_STRING; + if (parse_json_string(it, &it->string_value) < 0) + return -1; + break; + case '[': { + const char *save = it->begin; + size_t key_offset = it->key.len; + int i = 0, res; + + for (it->p++, skip_whitespace(it); *it->p != ']'; i++) { + strbuf_addf(&it->key, "[%d]", i); + + if ((res = iterate_json(it))) + return reset_iterator(it), res; + strbuf_setlen(&it->key, key_offset); + + skip_whitespace(it); + if (*it->p == ',') + it->p++; + } + + it->type = JSON_ARRAY; + it->begin = save; + it->end = it->p; + it->p++; + break; + } + case '{': { + const char *save = it->begin; + size_t key_offset = it->key.len; + int res; + + strbuf_addch(&it->key, '.'); + for (it->p++, skip_whitespace(it); *it->p != '}'; ) { + strbuf_setlen(&it->key, key_offset + 1); + if (parse_json_string(it, &it->key) < 0) + return -1; + skip_whitespace(it); + if (*(it->p)++ != ':') + return error("expected colon: %.*s", 5, it->p), + reset_iterator(it); + + if ((res = iterate_json(it))) + return res; + + skip_whitespace(it); + if (*it->p == ',') + it->p++; + } + strbuf_setlen(&it->key, key_offset); + + it->type = JSON_OBJECT; + it->begin = save; + it->end = it->p; + it->p++; + break; + } + } + + return it->fn(it); +} diff --git a/json-parser.h b/json-parser.h new file mode 100644 index 00000000000000..ce1fdc5ee23928 --- /dev/null +++ b/json-parser.h @@ -0,0 +1,29 @@ +#ifndef JSON_PARSER_H +#define JSON_PARSER_H + +#include "strbuf.h" + +struct json_iterator { + const char *json, *p, *begin, *end; + struct strbuf key, string_value; + enum { + JSON_NULL = 0, + JSON_FALSE, + JSON_TRUE, + JSON_NUMBER, + JSON_STRING, + JSON_ARRAY, + JSON_OBJECT + } type; + int (*fn)(struct json_iterator *it); + void *fn_data; +}; +#define JSON_ITERATOR_INIT(json_, fn_, fn_data_) { \ + .json = json_, .p = json_, \ + .key = STRBUF_INIT, .string_value = STRBUF_INIT, \ + .fn = fn_, .fn_data = fn_data_ \ +} + +int iterate_json(struct json_iterator *it); + +#endif From 6283bed830776f85a95ee72d0e0af43a5f5960b1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 24 Aug 2021 21:05:42 +0200 Subject: [PATCH 16/35] scalar clone: support GVFS-enabled remote repositories With this change, we come a big step closer to feature parity with Scalar: this allows cloning from Azure Repos (which do not support partial clones at time of writing). We use the just-implemented JSON parser to parse the response we got from the `gvfs/config` endpoint; Please note that this response might, or might not, contain information about a cache server. The presence or absence of said cache server, however, has nothing to do with the ability to speak the GVFS protocol (but the presence of the `gvfs/config` endpoint does that). An alternative considered during the development of this patch was to perform simple string matching instead of parsing the JSON-formatted data; However, this would have been fragile, as the response contains free-form text (e.g. the repository's description) which might contain parts that would confuse a simple string matcher (but not a proper JSON parser). Note: we need to limit the re-try logic in `git clone` to handle only the non-GVFS case: the call to `set_config()` to un-set the partial clone settings would otherwise fail because those settings would not exist in the GVFS protocol case. This will at least give us a clearer reason why such a fetch fails. Co-authored-by: Derrick Stolee Signed-off-by: Johannes Schindelin Signed-off-by: Derrick Stolee --- diagnose.c | 8 ++++ scalar.c | 126 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 131 insertions(+), 3 deletions(-) diff --git a/diagnose.c b/diagnose.c index beb0a8741ba3ce..2d4dc65fc3ffb2 100644 --- a/diagnose.c +++ b/diagnose.c @@ -7,6 +7,7 @@ #include "strvec.h" #include "object-store.h" #include "packfile.h" +#include "config.h" struct archive_dir { const char *path; @@ -172,6 +173,7 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode) struct strvec archiver_args = STRVEC_INIT; char **argv_copy = NULL; int stdout_fd = -1, archiver_fd = -1; + char *cache_server_url = NULL; struct strbuf buf = STRBUF_INIT; int res, i; struct archive_dir archive_dirs[] = { @@ -207,6 +209,11 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode) get_version_info(&buf, 1); strbuf_addf(&buf, "Repository root: %s\n", the_repository->worktree); + + git_config_get_string("gvfs.cache-server", &cache_server_url); + strbuf_addf(&buf, "Cache Server: %s\n\n", + cache_server_url ? cache_server_url : "None"); + get_disk_info(&buf); write_or_die(stdout_fd, buf.buf, buf.len); strvec_pushf(&archiver_args, @@ -264,6 +271,7 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode) free(argv_copy); strvec_clear(&archiver_args); strbuf_release(&buf); + free(cache_server_url); return res; } diff --git a/scalar.c b/scalar.c index cba905b36d1098..540ab160287d58 100644 --- a/scalar.c +++ b/scalar.c @@ -14,6 +14,7 @@ #include "dir.h" #include "packfile.h" #include "help.h" +#include "json-parser.h" static void setup_enlistment_directory(int argc, const char **argv, const char * const *usagestr, @@ -322,6 +323,80 @@ static int set_config(const char *fmt, ...) return res; } +/* Find N for which .CacheServers[N].GlobalDefault == true */ +static int get_cache_server_index(struct json_iterator *it) +{ + const char *p; + char *q; + long l; + + if (it->type == JSON_TRUE && + skip_iprefix(it->key.buf, ".CacheServers[", &p) && + (l = strtol(p, &q, 10)) >= 0 && p != q && + !strcasecmp(q, "].GlobalDefault")) { + *(long *)it->fn_data = l; + return 1; + } + + return 0; +} + +struct cache_server_url_data { + char *key, *url; +}; + +/* Get .CacheServers[N].Url */ +static int get_cache_server_url(struct json_iterator *it) +{ + struct cache_server_url_data *data = it->fn_data; + + if (it->type == JSON_STRING && + !strcasecmp(data->key, it->key.buf)) { + data->url = strbuf_detach(&it->string_value, NULL); + return 1; + } + + return 0; +} + +/* + * If `cache_server_url` is `NULL`, print the list to `stdout`. + * + * Since `gvfs-helper` requires a Git directory, this _must_ be run in + * a worktree. + */ +static int supports_gvfs_protocol(const char *url, char **cache_server_url) +{ + struct child_process cp = CHILD_PROCESS_INIT; + struct strbuf out = STRBUF_INIT; + + cp.git_cmd = 1; + strvec_pushl(&cp.args, "gvfs-helper", "--remote", url, "config", NULL); + if (!pipe_command(&cp, NULL, 0, &out, 512, NULL, 0)) { + long l = 0; + struct json_iterator it = + JSON_ITERATOR_INIT(out.buf, get_cache_server_index, &l); + struct cache_server_url_data data = { .url = NULL }; + + if (iterate_json(&it) < 0) { + strbuf_release(&out); + return error("JSON parse error"); + } + data.key = xstrfmt(".CacheServers[%ld].Url", l); + it.fn = get_cache_server_url; + it.fn_data = &data; + if (iterate_json(&it) < 0) { + strbuf_release(&out); + return error("JSON parse error"); + } + *cache_server_url = data.url; + free(data.key); + return 1; + } + strbuf_release(&out); + return 0; /* error out quietly */ +} + static char *remote_default_branch(const char *url) { struct child_process cp = CHILD_PROCESS_INIT; @@ -423,6 +498,8 @@ static int cmd_clone(int argc, const char **argv) { const char *branch = NULL; int full_clone = 0, single_branch = 0; + const char *cache_server_url = NULL; + char *default_cache_server_url = NULL; struct option clone_options[] = { OPT_STRING('b', "branch", &branch, N_(""), N_("branch to checkout after clone")), @@ -431,6 +508,9 @@ static int cmd_clone(int argc, const char **argv) OPT_BOOL(0, "single-branch", &single_branch, N_("only download metadata for the branch that will " "be checked out")), + OPT_STRING(0, "cache-server-url", &cache_server_url, + N_(""), + N_("the url or friendly name of the cache server")), OPT_END(), }; const char * const clone_usage[] = { @@ -441,6 +521,7 @@ static int cmd_clone(int argc, const char **argv) char *enlistment = NULL, *dir = NULL; struct strbuf buf = STRBUF_INIT; int res; + int gvfs_protocol; argc = parse_options(argc, argv, NULL, clone_options, clone_usage, 0); @@ -503,13 +584,46 @@ static int cmd_clone(int argc, const char **argv) set_config("remote.origin.fetch=" "+refs/heads/%s:refs/remotes/origin/%s", single_branch ? branch : "*", - single_branch ? branch : "*") || - set_config("remote.origin.promisor=true") || - set_config("remote.origin.partialCloneFilter=blob:none")) { + single_branch ? branch : "*")) { res = error(_("could not configure remote in '%s'"), dir); goto cleanup; } + if (set_config("credential.https://dev.azure.com.useHttpPath=true")) { + res = error(_("could not configure credential.useHttpPath")); + goto cleanup; + } + + gvfs_protocol = cache_server_url || + supports_gvfs_protocol(url, &default_cache_server_url); + + if (gvfs_protocol) { + if (!cache_server_url) + cache_server_url = default_cache_server_url; + if (set_config("core.useGVFSHelper=true") || + set_config("core.gvfs=150") || + set_config("http.version=HTTP/1.1")) { + res = error(_("could not turn on GVFS helper")); + goto cleanup; + } + if (cache_server_url && + set_config("gvfs.cache-server=%s", cache_server_url)) { + res = error(_("could not configure cache server")); + goto cleanup; + } + if (cache_server_url) + fprintf(stderr, "Cache server URL: %s\n", + cache_server_url); + } else { + if (set_config("core.useGVFSHelper=false") || + set_config("remote.origin.promisor=true") || + set_config("remote.origin.partialCloneFilter=blob:none")) { + res = error(_("could not configure partial clone in " + "'%s'"), dir); + goto cleanup; + } + } + if (!full_clone && (res = run_git("sparse-checkout", "init", "--cone", NULL))) goto cleanup; @@ -518,6 +632,11 @@ static int cmd_clone(int argc, const char **argv) return error(_("could not configure '%s'"), dir); if ((res = run_git("fetch", "--quiet", "origin", NULL))) { + if (gvfs_protocol) { + res = error(_("failed to prefetch commits and trees")); + goto cleanup; + } + warning(_("partial clone failed; attempting full clone")); if (set_config("remote.origin.promisor") || @@ -548,6 +667,7 @@ static int cmd_clone(int argc, const char **argv) free(enlistment); free(dir); strbuf_release(&buf); + free(default_cache_server_url); return res; } From 75172a6f5a626c744150407092648ebd23dd35d4 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 16 Apr 2021 19:47:05 +0200 Subject: [PATCH 17/35] test-gvfs-protocol: also serve smart protocol This comes in handy, as we want to verify that `scalar clone` also works against a GVFS-enabled remote repository. Note that we have to set `MSYS2_ENV_CONV_EXCL` to prevent MSYS2 from mangling `PATH_TRANSLATED`: The value _does_ look like a Unix-style path, but no, MSYS2 must not be allowed to convert that into a Windows path: `http-backend` needs it in the unmodified form. (The MSYS2 runtime comes in when `git` is run via `bin-wrappers/git`, which is a shell script.) Signed-off-by: Johannes Schindelin --- t/helper/test-gvfs-protocol.c | 49 +++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/t/helper/test-gvfs-protocol.c b/t/helper/test-gvfs-protocol.c index 97b3d363bf3fa5..a437248a10feff 100644 --- a/t/helper/test-gvfs-protocol.c +++ b/t/helper/test-gvfs-protocol.c @@ -1481,6 +1481,8 @@ static enum worker_result req__read(struct req *req, int fd) static enum worker_result dispatch(struct req *req) { + static regex_t *smart_http_regex; + static int initialized; const char *method; enum worker_result wr; @@ -1529,6 +1531,53 @@ static enum worker_result dispatch(struct req *req) return do__gvfs_prefetch__get(req); } + if (!initialized) { + smart_http_regex = xmalloc(sizeof(*smart_http_regex)); + if (regcomp(smart_http_regex, "^/(HEAD|info/refs|" + "objects/info/[^/]+|git-(upload|receive)-pack)$", + REG_EXTENDED)) { + warning("could not compile smart HTTP regex"); + smart_http_regex = NULL; + } + initialized = 1; + } + + if (smart_http_regex && + !regexec(smart_http_regex, req->uri_base.buf, 0, NULL, 0)) { + const char *ok = "HTTP/1.1 200 OK\r\n"; + struct child_process cp = CHILD_PROCESS_INIT; + int i, res; + + if (write(1, ok, strlen(ok)) < 0) + return error(_("could not send '%s'"), ok); + + strvec_pushf(&cp.env, "REQUEST_METHOD=%s", method); + strvec_pushf(&cp.env, "PATH_TRANSLATED=%s", + req->uri_base.buf); + /* Prevent MSYS2 from "converting to a Windows path" */ + strvec_pushf(&cp.env, + "MSYS2_ENV_CONV_EXCL=PATH_TRANSLATED"); + strvec_push(&cp.env, "SERVER_PROTOCOL=HTTP/1.1"); + if (req->quest_args.len) + strvec_pushf(&cp.env, "QUERY_STRING=%s", + req->quest_args.buf); + for (i = 0; i < req->header_list.nr; i++) { + const char *header = req->header_list.items[i].string; + if (!strncasecmp("Content-Type: ", header, 14)) + strvec_pushf(&cp.env, "CONTENT_TYPE=%s", + header + 14); + else if (!strncasecmp("Content-Length: ", header, 16)) + strvec_pushf(&cp.env, "CONTENT_LENGTH=%s", + header + 16); + } + cp.git_cmd = 1; + strvec_push(&cp.args, "http-backend"); + res = run_command(&cp); + close(1); + close(0); + return !!res; + } + return send_http_error(1, 501, "Not Implemented", -1, WR_OK | WR_HANGUP); } From dd91927ed3398456e246eb57a4b2dbb21721db71 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 26 Apr 2021 17:31:40 +0200 Subject: [PATCH 18/35] gvfs-helper: add the `endpoint` command We already have the `config` command that accesses the `gvfs/config` endpoint. To implement `scalar`, we also need to be able to access the `vsts/info` endpoint. Let's add a command to do precisely that. Signed-off-by: Johannes Schindelin --- gvfs-helper.c | 61 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 7 deletions(-) diff --git a/gvfs-helper.c b/gvfs-helper.c index ce89edc33cebe3..e8e6fbbdd9f361 100644 --- a/gvfs-helper.c +++ b/gvfs-helper.c @@ -202,6 +202,12 @@ // [2] Documentation/technical/long-running-process-protocol.txt // [3] See GIT_TRACE_PACKET // +// endpoint +// +// Fetch the given endpoint from the main Git server (specifying +// `gvfs/config` as endpoint is idempotent to the `config` +// command mentioned above). +// ////////////////////////////////////////////////////////////////// #include "cache.h" @@ -3110,18 +3116,20 @@ static void do_req__with_fallback(const char *url_component, * * Return server's response buffer. This is probably a raw JSON string. */ -static void do__http_get__gvfs_config(struct gh__response_status *status, - struct strbuf *config_data) +static void do__http_get__simple_endpoint(struct gh__response_status *status, + struct strbuf *response, + const char *endpoint, + const char *tr2_label) { struct gh__request_params params = GH__REQUEST_PARAMS_INIT; - strbuf_addstr(¶ms.tr2_label, "GET/config"); + strbuf_addstr(¶ms.tr2_label, tr2_label); params.b_is_post = 0; params.b_write_to_file = 0; /* cache-servers do not handle gvfs/config REST calls */ params.b_permit_cache_server_if_defined = 0; - params.buffer = config_data; + params.buffer = response; params.objects_mode = GH__OBJECTS_MODE__NONE; params.object_count = 1; /* a bit of a lie */ @@ -3143,15 +3151,22 @@ static void do__http_get__gvfs_config(struct gh__response_status *status, * see any need to report progress on the upload side of * the GET. So just report progress on the download side. */ - strbuf_addstr(¶ms.progress_base_phase3_msg, - "Receiving gvfs/config"); + strbuf_addf(¶ms.progress_base_phase3_msg, + "Receiving %s", endpoint); } - do_req__with_fallback("gvfs/config", ¶ms, status); + do_req__with_fallback(endpoint, ¶ms, status); gh__request_params__release(¶ms); } +static void do__http_get__gvfs_config(struct gh__response_status *status, + struct strbuf *config_data) +{ + do__http_get__simple_endpoint(status, config_data, "gvfs/config", + "GET/config"); +} + static void setup_gvfs_objects_progress(struct gh__request_params *params, unsigned long num, unsigned long den) { @@ -3596,6 +3611,35 @@ static enum gh__error_code do_sub_cmd__config(int argc, const char **argv) return ec; } +static enum gh__error_code do_sub_cmd__endpoint(int argc, const char **argv) +{ + struct gh__response_status status = GH__RESPONSE_STATUS_INIT; + struct strbuf data = STRBUF_INIT; + enum gh__error_code ec = GH__ERROR_CODE__OK; + const char *endpoint; + + if (argc != 2) + return GH__ERROR_CODE__ERROR; + endpoint = argv[1]; + + trace2_cmd_mode(endpoint); + + finish_init(0); + + do__http_get__simple_endpoint(&status, &data, endpoint, endpoint); + ec = status.ec; + + if (ec == GH__ERROR_CODE__OK) + printf("%s\n", data.buf); + else + error("config: %s", status.error_message.buf); + + gh__response_status__release(&status); + strbuf_release(&data); + + return ec; +} + /* * Read a list of objects from stdin and fetch them as a series of * single object HTTP GET requests. @@ -4087,6 +4131,9 @@ static enum gh__error_code do_sub_cmd(int argc, const char **argv) if (!strcmp(argv[0], "config")) return do_sub_cmd__config(argc, argv); + if (!strcmp(argv[0], "endpoint")) + return do_sub_cmd__endpoint(argc, argv); + if (!strcmp(argv[0], "prefetch")) return do_sub_cmd__prefetch(argc, argv); From 40d0536a2e1512ea3f4ca1b6de266eed5d1a93bf Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 15 May 2021 00:04:20 +0200 Subject: [PATCH 19/35] dir_inside_of(): handle directory separators correctly On Windows, both the forward slash and the backslash are directory separators. Which means that `a\b\c` really is inside `a/b`. Therefore, we need to special-case the directory separators in the helper function `cmp_icase()` that is used in the loop in `dir_inside_of()`. Signed-off-by: Johannes Schindelin --- dir.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dir.c b/dir.c index 6f9fbb28852c95..c3a32efd47044c 100644 --- a/dir.c +++ b/dir.c @@ -3122,6 +3122,8 @@ static int cmp_icase(char a, char b) { if (a == b) return 0; + if (is_dir_sep(a)) + return is_dir_sep(b) ? 0 : -1; if (ignore_case) return toupper(a) - toupper(b); return a - b; From cee15c6d03dd769ad47ad0a6e6102707580e754e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 6 May 2021 14:35:12 +0200 Subject: [PATCH 20/35] scalar: disable authentication in unattended mode Modified to remove call to is_unattended() that has not been implemented yet. Signed-off-by: Johannes Schindelin --- t/t9099-scalar.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/t/t9099-scalar.sh b/t/t9099-scalar.sh index 62b92d361e2bfd..886c32ece1cccd 100755 --- a/t/t9099-scalar.sh +++ b/t/t9099-scalar.sh @@ -7,6 +7,13 @@ test_description='test the `scalar` command' GIT_TEST_MAINT_SCHEDULER="crontab:test-tool crontab ../cron.txt,launchctl:true,schtasks:true" export GIT_TEST_MAINT_SCHEDULER +# Do not write any files outside the trash directory +Scalar_UNATTENDED=1 +export Scalar_UNATTENDED + +GIT_ASKPASS=true +export GIT_ASKPASS + test_expect_success 'scalar shows a usage' ' test_expect_code 129 scalar -h ' From 8c30c979d0f198c7513047d298a11cb5682190a0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 3 May 2021 23:41:28 +0200 Subject: [PATCH 21/35] scalar: do initialize `gvfs.sharedCache` This finalizes the port of the `QueryVstsInfo()` function: we already taught `gvfs-helper` to access the `vsts/info` endpoint on demand, we implemented proper JSON parsing, and now it is time to hook it all up. To that end, we also provide a default local cache root directory. It works the same way as the .NET version of Scalar: it uses C:\scalarCache on Windows, ~/.scalarCache/ on macOS and ~/.cache/scalar on Linux Modified to include call to is_unattended() that was removed from a previous commit. Signed-off-by: Johannes Schindelin --- Documentation/scalar.txt | 15 +++- diagnose.c | 9 +- scalar.c | 182 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 200 insertions(+), 6 deletions(-) diff --git a/Documentation/scalar.txt b/Documentation/scalar.txt index f33436c7f65ff9..e7f6d393974436 100644 --- a/Documentation/scalar.txt +++ b/Documentation/scalar.txt @@ -8,7 +8,9 @@ scalar - A tool for managing large Git repositories SYNOPSIS -------- [verse] -scalar clone [--single-branch] [--branch ] [--full-clone] [] +scalar clone [--single-branch] [--branch ] [--full-clone] + [--local-cache-path ] [--cache-server-url ] + [] scalar list scalar register [] scalar unregister [] @@ -84,6 +86,17 @@ cloning. If the HEAD at the remote did not point at any branch when A sparse-checkout is initialized by default. This behavior can be turned off via `--full-clone`. +--local-cache-path :: + Override the path to the local cache root directory; Pre-fetched objects + are stored into a repository-dependent subdirectory of that path. ++ +The default is `:\.scalarCache` on Windows (on the same drive as the +clone), and `~/.scalarCache` on macOS. + +--cache-server-url :: + Retrieve missing objects from the specified remote, which is expected to + understand the GVFS protocol. + List ~~~~ diff --git a/diagnose.c b/diagnose.c index 2d4dc65fc3ffb2..cc6d7fe351ee55 100644 --- a/diagnose.c +++ b/diagnose.c @@ -173,7 +173,7 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode) struct strvec archiver_args = STRVEC_INIT; char **argv_copy = NULL; int stdout_fd = -1, archiver_fd = -1; - char *cache_server_url = NULL; + char *cache_server_url = NULL, *shared_cache = NULL; struct strbuf buf = STRBUF_INIT; int res, i; struct archive_dir archive_dirs[] = { @@ -211,8 +211,10 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode) strbuf_addf(&buf, "Repository root: %s\n", the_repository->worktree); git_config_get_string("gvfs.cache-server", &cache_server_url); - strbuf_addf(&buf, "Cache Server: %s\n\n", - cache_server_url ? cache_server_url : "None"); + git_config_get_string("gvfs.sharedCache", &shared_cache); + strbuf_addf(&buf, "Cache Server: %s\nLocal Cache: %s\n\n", + cache_server_url ? cache_server_url : "None", + shared_cache ? shared_cache : "None"); get_disk_info(&buf); write_or_die(stdout_fd, buf.buf, buf.len); @@ -272,6 +274,7 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode) strvec_clear(&archiver_args); strbuf_release(&buf); free(cache_server_url); + free(shared_cache); return res; } diff --git a/scalar.c b/scalar.c index 540ab160287d58..04d27c7c931f80 100644 --- a/scalar.c +++ b/scalar.c @@ -16,6 +16,10 @@ #include "help.h" #include "json-parser.h" +static int is_unattended(void) { + return git_env_bool("Scalar_UNATTENDED", 0); +} + static void setup_enlistment_directory(int argc, const char **argv, const char * const *usagestr, const struct option *options, @@ -95,6 +99,19 @@ static int run_git(const char *arg, ...) return res; } +static const char *ensure_absolute_path(const char *path, char **absolute) +{ + struct strbuf buf = STRBUF_INIT; + + if (is_absolute_path(path)) + return path; + + strbuf_realpath_forgiving(&buf, path, 1); + free(*absolute); + *absolute = strbuf_detach(&buf, NULL); + return *absolute; +} + struct scalar_config { const char *key; const char *value; @@ -397,6 +414,87 @@ static int supports_gvfs_protocol(const char *url, char **cache_server_url) return 0; /* error out quietly */ } +static char *default_cache_root(const char *root) +{ + const char *env; + + if (is_unattended()) + return xstrfmt("%s/.scalarCache", root); + +#ifdef WIN32 + (void)env; + return xstrfmt("%.*s.scalarCache", offset_1st_component(root), root); +#elif defined(__APPLE__) + if ((env = getenv("HOME")) && *env) + return xstrfmt("%s/.scalarCache", env); + return NULL; +#else + if ((env = getenv("XDG_CACHE_HOME")) && *env) + return xstrfmt("%s/scalar", env); + if ((env = getenv("HOME")) && *env) + return xstrfmt("%s/.cache/scalar", env); + return NULL; +#endif +} + +static int get_repository_id(struct json_iterator *it) +{ + if (it->type == JSON_STRING && + !strcasecmp(".repository.id", it->key.buf)) { + *(char **)it->fn_data = strbuf_detach(&it->string_value, NULL); + return 1; + } + + return 0; +} + +/* Needs to run this in a worktree; gvfs-helper requires a Git repository */ +static char *get_cache_key(const char *url) +{ + struct child_process cp = CHILD_PROCESS_INIT; + struct strbuf out = STRBUF_INIT; + char *cache_key = NULL; + + cp.git_cmd = 1; + strvec_pushl(&cp.args, "gvfs-helper", "--remote", url, + "endpoint", "vsts/info", NULL); + if (!pipe_command(&cp, NULL, 0, &out, 512, NULL, 0)) { + char *id = NULL; + struct json_iterator it = + JSON_ITERATOR_INIT(out.buf, get_repository_id, &id); + + if (iterate_json(&it) < 0) + warning("JSON parse error (%s)", out.buf); + else if (id) + cache_key = xstrfmt("id_%s", id); + free(id); + } + + if (!cache_key) { + struct strbuf downcased = STRBUF_INIT; + int hash_algo_index = hash_algo_by_name("sha1"); + const struct git_hash_algo *hash_algo = hash_algo_index < 0 ? + the_hash_algo : &hash_algos[hash_algo_index]; + git_hash_ctx ctx; + unsigned char hash[GIT_MAX_RAWSZ]; + + strbuf_addstr(&downcased, url); + strbuf_tolower(&downcased); + + hash_algo->init_fn(&ctx); + hash_algo->update_fn(&ctx, downcased.buf, downcased.len); + hash_algo->final_fn(hash, &ctx); + + strbuf_release(&downcased); + + cache_key = xstrfmt("url_%s", + hash_to_hex_algop(hash, hash_algo)); + } + + strbuf_release(&out); + return cache_key; +} + static char *remote_default_branch(const char *url) { struct child_process cp = CHILD_PROCESS_INIT; @@ -494,12 +592,48 @@ void load_builtin_commands(const char *prefix, struct cmdnames *cmds) die("not implemented"); } +static int init_shared_object_cache(const char *url, + const char *local_cache_root) +{ + struct strbuf buf = STRBUF_INIT; + int res = 0; + char *cache_key = NULL, *shared_cache_path = NULL; + + if (!(cache_key = get_cache_key(url))) { + res = error(_("could not determine cache key for '%s'"), url); + goto cleanup; + } + + shared_cache_path = xstrfmt("%s/%s", local_cache_root, cache_key); + if (set_config("gvfs.sharedCache=%s", shared_cache_path)) { + res = error(_("could not configure shared cache")); + goto cleanup; + } + + strbuf_addf(&buf, "%s/pack", shared_cache_path); + switch (safe_create_leading_directories(buf.buf)) { + case SCLD_OK: case SCLD_EXISTS: + break; /* okay */ + default: + res = error_errno(_("could not initialize '%s'"), buf.buf); + goto cleanup; + } + + write_file(git_path("objects/info/alternates"),"%s\n", shared_cache_path); + + cleanup: + strbuf_release(&buf); + free(shared_cache_path); + free(cache_key); + return res; +} + static int cmd_clone(int argc, const char **argv) { const char *branch = NULL; int full_clone = 0, single_branch = 0; - const char *cache_server_url = NULL; - char *default_cache_server_url = NULL; + const char *cache_server_url = NULL, *local_cache_root = NULL; + char *default_cache_server_url = NULL, *local_cache_root_abs = NULL; struct option clone_options[] = { OPT_STRING('b', "branch", &branch, N_(""), N_("branch to checkout after clone")), @@ -511,6 +645,9 @@ static int cmd_clone(int argc, const char **argv) OPT_STRING(0, "cache-server-url", &cache_server_url, N_(""), N_("the url or friendly name of the cache server")), + OPT_STRING(0, "local-cache-path", &local_cache_root, + N_(""), + N_("override the path for the local Scalar cache")), OPT_END(), }; const char * const clone_usage[] = { @@ -551,8 +688,20 @@ static int cmd_clone(int argc, const char **argv) if (is_directory(enlistment)) die(_("directory '%s' exists already"), enlistment); + ensure_absolute_path(enlistment, &enlistment); + dir = xstrfmt("%s/src", enlistment); + if (!local_cache_root) + local_cache_root = local_cache_root_abs = + default_cache_root(enlistment); + else + local_cache_root = ensure_absolute_path(local_cache_root, + &local_cache_root_abs); + + if (!local_cache_root) + die(_("could not determine local cache root")); + strbuf_reset(&buf); if (branch) strbuf_addf(&buf, "init.defaultBranch=%s", branch); @@ -572,8 +721,28 @@ static int cmd_clone(int argc, const char **argv) setup_git_directory(); + git_config(git_default_config, NULL); + + /* + * This `dir_inside_of()` call relies on git_config() having parsed the + * newly-initialized repository config's `core.ignoreCase` value. + */ + if (dir_inside_of(local_cache_root, dir) >= 0) { + struct strbuf path = STRBUF_INIT; + + strbuf_addstr(&path, enlistment); + if (chdir("../..") < 0 || + remove_dir_recursively(&path, 0) < 0) + die(_("'--local-cache-path' cannot be inside the src " + "folder;\nCould not remove '%s'"), enlistment); + + die(_("'--local-cache-path' cannot be inside the src folder")); + } + /* common-main already logs `argv` */ trace2_def_repo(the_repository); + trace2_data_intmax("scalar", the_repository, "unattended", + is_unattended()); if (!branch && !(branch = remote_default_branch(url))) { res = error(_("failed to get default branch for '%s'"), url); @@ -598,6 +767,8 @@ static int cmd_clone(int argc, const char **argv) supports_gvfs_protocol(url, &default_cache_server_url); if (gvfs_protocol) { + if ((res = init_shared_object_cache(url, local_cache_root))) + goto cleanup; if (!cache_server_url) cache_server_url = default_cache_server_url; if (set_config("core.useGVFSHelper=true") || @@ -668,6 +839,7 @@ static int cmd_clone(int argc, const char **argv) free(dir); strbuf_release(&buf); free(default_cache_server_url); + free(local_cache_root_abs); return res; } @@ -1026,6 +1198,12 @@ int cmd_main(int argc, const char **argv) struct strbuf scalar_usage = STRBUF_INIT; int i; + if (is_unattended()) { + setenv("GIT_ASKPASS", "", 0); + setenv("GIT_TERMINAL_PROMPT", "false", 0); + git_config_push_parameter("credential.interactive=never"); + } + while (argc > 1 && *argv[1] == '-') { if (!strcmp(argv[1], "-C")) { if (argc < 3) From 7fdea19837b03e91dc321891020d51c5d39ff5b1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 1 Jun 2021 23:18:14 +0200 Subject: [PATCH 22/35] scalar diagnose: include shared cache info Signed-off-by: Johannes Schindelin --- diagnose.c | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/diagnose.c b/diagnose.c index cc6d7fe351ee55..659792473b4239 100644 --- a/diagnose.c +++ b/diagnose.c @@ -67,6 +67,39 @@ static int dir_file_stats(struct object_directory *object_dir, void *data) return 0; } +static void dir_stats(struct strbuf *buf, const char *path) +{ + DIR *dir = opendir(path); + struct dirent *e; + struct stat e_stat; + struct strbuf file_path = STRBUF_INIT; + size_t base_path_len; + + if (!dir) + return; + + strbuf_addstr(buf, "Contents of "); + strbuf_add_absolute_path(buf, path); + strbuf_addstr(buf, ":\n"); + + strbuf_add_absolute_path(&file_path, path); + strbuf_addch(&file_path, '/'); + base_path_len = file_path.len; + + while ((e = readdir(dir)) != NULL) + if (!is_dot_or_dotdot(e->d_name) && e->d_type == DT_REG) { + strbuf_setlen(&file_path, base_path_len); + strbuf_addstr(&file_path, e->d_name); + if (!stat(file_path.buf, &e_stat)) + strbuf_addf(buf, "%-70s %16"PRIuMAX"\n", + e->d_name, + (uintmax_t)e_stat.st_size); + } + + strbuf_release(&file_path); + closedir(dir); +} + static int count_files(char *path) { DIR *dir = opendir(path); @@ -174,7 +207,7 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode) char **argv_copy = NULL; int stdout_fd = -1, archiver_fd = -1; char *cache_server_url = NULL, *shared_cache = NULL; - struct strbuf buf = STRBUF_INIT; + struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT; int res, i; struct archive_dir archive_dirs[] = { { ".git", 0 }, @@ -246,6 +279,41 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode) } } + if (shared_cache) { + strbuf_reset(&buf); + strbuf_addf(&path, "%s/pack", shared_cache); + strbuf_reset(&buf); + strbuf_addstr(&buf, "--add-virtual-file=packs-cached.txt:"); + dir_stats(&buf, path.buf); + strvec_push(&archiver_args, buf.buf); + + strbuf_reset(&buf); + strbuf_addstr(&buf, "--add-virtual-file=objects-cached.txt:"); + loose_objs_stats(&buf, shared_cache); + strvec_push(&archiver_args, buf.buf); + + strbuf_reset(&path); + strbuf_addf(&path, "%s/info", shared_cache); + if (is_directory(path.buf)) { + DIR *dir = opendir(path.buf); + struct dirent *e; + + while ((e = readdir(dir))) { + if (!strcmp(".", e->d_name) || !strcmp("..", e->d_name)) + continue; + + strbuf_reset(&buf); + strbuf_addf(&buf, "--add-virtual-file=info/%s:", e->d_name); + if (strbuf_read_file(&buf, path.buf, 0) < 0) { + res = error_errno(_("could not read '%s'"), path.buf); + goto diagnose_cleanup; + } + strvec_push(&archiver_args, buf.buf); + } + closedir(dir); + } + } + strvec_pushl(&archiver_args, "--prefix=", oid_to_hex(the_hash_algo->empty_tree), "--", NULL); From 07dcf554eb4394ffa9f300f4312263b8e5b4aacb Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 28 Apr 2021 13:56:16 +0200 Subject: [PATCH 23/35] scalar: only try GVFS protocol on https:// URLs Well, technically also the http:// protocol is allowed _when testing_... Signed-off-by: Johannes Schindelin --- scalar.c | 47 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/scalar.c b/scalar.c index 04d27c7c931f80..e57b08af27fe3a 100644 --- a/scalar.c +++ b/scalar.c @@ -376,6 +376,13 @@ static int get_cache_server_url(struct json_iterator *it) return 0; } +static int can_url_support_gvfs(const char *url) +{ + return starts_with(url, "https://") || + (git_env_bool("GIT_TEST_ALLOW_GVFS_VIA_HTTP", 0) && + starts_with(url, "http://")); +} + /* * If `cache_server_url` is `NULL`, print the list to `stdout`. * @@ -387,6 +394,13 @@ static int supports_gvfs_protocol(const char *url, char **cache_server_url) struct child_process cp = CHILD_PROCESS_INIT; struct strbuf out = STRBUF_INIT; + /* + * The GVFS protocol is only supported via https://; For testing, we + * also allow http://. + */ + if (!can_url_support_gvfs(url)) + return 0; + cp.git_cmd = 1; strvec_pushl(&cp.args, "gvfs-helper", "--remote", url, "config", NULL); if (!pipe_command(&cp, NULL, 0, &out, 512, NULL, 0)) { @@ -455,19 +469,26 @@ static char *get_cache_key(const char *url) struct strbuf out = STRBUF_INIT; char *cache_key = NULL; - cp.git_cmd = 1; - strvec_pushl(&cp.args, "gvfs-helper", "--remote", url, - "endpoint", "vsts/info", NULL); - if (!pipe_command(&cp, NULL, 0, &out, 512, NULL, 0)) { - char *id = NULL; - struct json_iterator it = - JSON_ITERATOR_INIT(out.buf, get_repository_id, &id); - - if (iterate_json(&it) < 0) - warning("JSON parse error (%s)", out.buf); - else if (id) - cache_key = xstrfmt("id_%s", id); - free(id); + /* + * The GVFS protocol is only supported via https://; For testing, we + * also allow http://. + */ + if (can_url_support_gvfs(url)) { + cp.git_cmd = 1; + strvec_pushl(&cp.args, "gvfs-helper", "--remote", url, + "endpoint", "vsts/info", NULL); + if (!pipe_command(&cp, NULL, 0, &out, 512, NULL, 0)) { + char *id = NULL; + struct json_iterator it = + JSON_ITERATOR_INIT(out.buf, get_repository_id, + &id); + + if (iterate_json(&it) < 0) + warning("JSON parse error (%s)", out.buf); + else if (id) + cache_key = xstrfmt("id_%s", id); + free(id); + } } if (!cache_key) { From 7f0e20b02a7ece0491bf22a62a477ce39e1a543a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 16 Apr 2021 21:43:57 +0200 Subject: [PATCH 24/35] scalar: verify that we can use a GVFS-enabled repository Azure Repos does not support partial clones at the moment, but it does support the GVFS protocol. To that end, the Microsoft fork of Git has a `gvfs-helper` command that is optionally used to perform essentially the same functionality as partial clone. Let's verify that `scalar clone` detects that situation and enables the GVFS helper. Signed-off-by: Johannes Schindelin --- t/t9099-scalar.sh | 137 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/t/t9099-scalar.sh b/t/t9099-scalar.sh index 886c32ece1cccd..9870f2f04994a3 100755 --- a/t/t9099-scalar.sh +++ b/t/t9099-scalar.sh @@ -214,4 +214,141 @@ test_expect_success UNZIP 'scalar diagnose' ' grep "^Total: [1-9]" out ' +GIT_TEST_ALLOW_GVFS_VIA_HTTP=1 +export GIT_TEST_ALLOW_GVFS_VIA_HTTP + +test_set_port GIT_TEST_GVFS_PROTOCOL_PORT +HOST_PORT=127.0.0.1:$GIT_TEST_GVFS_PROTOCOL_PORT +PID_FILE="$(pwd)"/pid-file.pid +SERVER_LOG="$(pwd)"/OUT.server.log + +test_atexit ' + test -f "$PID_FILE" || return 0 + + # The server will shutdown automatically when we delete the pid-file. + rm -f "$PID_FILE" + + test -z "$verbose$verbose_log" || { + echo "server log:" + cat "$SERVER_LOG" + } + + # Give it a few seconds to shutdown (mainly to completely release the + # port before the next test start another instance and it attempts to + # bind to it). + for k in $(test_seq 5) + do + grep -q "Starting graceful shutdown" "$SERVER_LOG" && + return 0 || + sleep 1 + done + + echo "stop_gvfs_protocol_server: timeout waiting for server shutdown" + return 1 +' + +start_gvfs_enabled_http_server () { + GIT_HTTP_EXPORT_ALL=1 \ + test-gvfs-protocol --verbose \ + --listen=127.0.0.1 \ + --port=$GIT_TEST_GVFS_PROTOCOL_PORT \ + --reuseaddr \ + --pid-file="$PID_FILE" \ + 2>"$SERVER_LOG" & + + for k in 0 1 2 3 4 + do + if test -f "$PID_FILE" + then + return 0 + fi + sleep 1 + done + return 1 +} + +test_expect_success 'start GVFS-enabled server' ' + git config uploadPack.allowFilter false && + git config uploadPack.allowAnySHA1InWant false && + start_gvfs_enabled_http_server +' + +test_expect_success '`scalar clone` with GVFS-enabled server' ' + : the fake cache server requires fake authentication && + git config --global core.askPass true && + scalar clone --single-branch -- http://$HOST_PORT/ using-gvfs && + + : verify that the shared cache has been configured && + cache_key="url_$(printf "%s" http://$HOST_PORT/ | + tr A-Z a-z | + test-tool sha1)" && + echo "$(pwd)/using-gvfs/.scalarCache/$cache_key" >expect && + git -C using-gvfs/src config gvfs.sharedCache >actual && + test_cmp expect actual && + + second=$(git rev-parse --verify second:second.t) && + ( + cd using-gvfs/src && + test_path_is_missing 1/2 && + GIT_TRACE=$PWD/trace.txt git cat-file blob $second >actual && + : verify that the gvfs-helper was invoked to fetch it && + test_i18ngrep gvfs-helper trace.txt && + echo "second" >expect && + test_cmp expect actual + ) +' + +test_expect_success '`scalar register` parallel to worktree' ' + git init test-repo/src && + mkdir -p test-repo/out && + scalar register test-repo/out && + git config --get --global --fixed-value \ + maintenance.repo "$(pwd)/test-repo/src" && + scalar list >scalar.repos && + grep -F "$(pwd)/test-repo/src" scalar.repos && + scalar delete test-repo +' + +test_expect_success '`scalar register` & `unregister` with existing repo' ' + git init existing && + scalar register existing && + git config --get --global --fixed-value \ + maintenance.repo "$(pwd)/existing" && + scalar list >scalar.repos && + grep -F "$(pwd)/existing" scalar.repos && + scalar unregister existing && + test_must_fail git config --get --global --fixed-value \ + maintenance.repo "$(pwd)/existing" && + scalar list >scalar.repos && + ! grep -F "$(pwd)/existing" scalar.repos +' + +test_expect_success '`scalar unregister` with existing repo, deleted .git' ' + scalar register existing && + rm -rf existing/.git && + scalar unregister existing && + test_must_fail git config --get --global --fixed-value \ + maintenance.repo "$(pwd)/existing" && + scalar list >scalar.repos && + ! grep -F "$(pwd)/existing" scalar.repos +' + +test_expect_success '`scalar register` existing repo with `src` folder' ' + git init existing && + mkdir -p existing/src && + scalar register existing/src && + scalar list >scalar.repos && + grep -F "$(pwd)/existing" scalar.repos && + scalar unregister existing && + scalar list >scalar.repos && + ! grep -F "$(pwd)/existing" scalar.repos +' + +test_expect_success '`scalar delete` with existing repo' ' + git init existing && + scalar register existing && + scalar delete existing && + test_path_is_missing existing +' + test_done From 6f590631d7230a2098a9a0eb67e7c37ed619f44c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 23 Apr 2021 16:12:33 +0200 Subject: [PATCH 25/35] scalar: add the `cache-server` command This allows setting the GVFS-enabled cache server, or listing the one(s) associated with the remote repository. Signed-off-by: Johannes Schindelin --- Documentation/scalar.txt | 22 +++++++++ scalar.c | 102 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/Documentation/scalar.txt b/Documentation/scalar.txt index e7f6d393974436..0424d020d9b9f4 100644 --- a/Documentation/scalar.txt +++ b/Documentation/scalar.txt @@ -18,6 +18,7 @@ scalar run ( all | config | commit-graph | fetch | loose-objects | pack-files ) scalar reconfigure [ --all | ] scalar diagnose [] scalar delete +scalar cache-server ( --get | --set | --list [] ) [] DESCRIPTION ----------- @@ -170,6 +171,27 @@ delete :: This subcommand lets you delete an existing Scalar enlistment from your local file system, unregistering the repository. +Cache-server +~~~~~~~~~~~~ + +cache-server ( --get | --set | --list [] ) []:: + This command lets you query or set the GVFS-enabled cache server used + to fetch missing objects. + +--get:: + This is the default command mode: query the currently-configured cache + server URL, if any. + +--list:: + Access the `gvfs/info` endpoint of the specified remote (default: + `origin`) to figure out which cache servers are available, if any. ++ +In contrast to the `--get` command mode (which only accesses the local +repository), this command mode triggers a request via the network that +potentially requires authentication. If authentication is required, the +configured credential helper is employed (see linkgit:git-credential[1] +for details). + SEE ALSO -------- linkgit:git-clone[1], linkgit:git-maintenance[1]. diff --git a/scalar.c b/scalar.c index e57b08af27fe3a..cd625c3564a120 100644 --- a/scalar.c +++ b/scalar.c @@ -15,6 +15,7 @@ #include "packfile.h" #include "help.h" #include "json-parser.h" +#include "remote.h" static int is_unattended(void) { return git_env_bool("Scalar_UNATTENDED", 0); @@ -340,6 +341,21 @@ static int set_config(const char *fmt, ...) return res; } +static int list_cache_server_urls(struct json_iterator *it) +{ + const char *p; + char *q; + long l; + + if (it->type == JSON_STRING && + skip_iprefix(it->key.buf, ".CacheServers[", &p) && + (l = strtol(p, &q, 10)) >= 0 && p != q && + !strcasecmp(q, "].Url")) + printf("#%ld: %s\n", l, it->string_value.buf); + + return 0; +} + /* Find N for which .CacheServers[N].GlobalDefault == true */ static int get_cache_server_index(struct json_iterator *it) { @@ -409,6 +425,16 @@ static int supports_gvfs_protocol(const char *url, char **cache_server_url) JSON_ITERATOR_INIT(out.buf, get_cache_server_index, &l); struct cache_server_url_data data = { .url = NULL }; + if (!cache_server_url) { + it.fn = list_cache_server_urls; + if (iterate_json(&it) < 0) { + strbuf_release(&out); + return error("JSON parse error"); + } + strbuf_release(&out); + return 0; + } + if (iterate_json(&it) < 0) { strbuf_release(&out); return error("JSON parse error"); @@ -425,7 +451,9 @@ static int supports_gvfs_protocol(const char *url, char **cache_server_url) return 1; } strbuf_release(&out); - return 0; /* error out quietly */ + /* error out quietly, unless we wanted to list URLs */ + return cache_server_url ? + 0 : error(_("Could not access gvfs/config endpoint")); } static char *default_cache_root(const char *root) @@ -1197,6 +1225,77 @@ static int cmd_version(int argc, const char **argv) return 0; } +static int cmd_cache_server(int argc, const char **argv) +{ + int get = 0; + char *set = NULL, *list = NULL; + const char *default_remote = "(default)"; + struct option options[] = { + OPT_BOOL(0, "get", &get, + N_("get the configured cache-server URL")), + OPT_STRING(0, "set", &set, N_("URL"), + N_("configure the cache-server to use")), + { OPTION_STRING, 0, "list", &list, N_("remote"), + N_("list the possible cache-server URLs"), + PARSE_OPT_OPTARG, NULL, (intptr_t) default_remote }, + OPT_END(), + }; + const char * const usage[] = { + N_("scalar cache_server " + "[--get | --set | --list []] []"), + NULL + }; + int res = 0; + + argc = parse_options(argc, argv, NULL, options, + usage, 0); + + if (get + !!set + !!list > 1) + usage_msg_opt(_("--get/--set/--list are mutually exclusive"), + usage, options); + + setup_enlistment_directory(argc, argv, usage, options, NULL); + + if (list) { + const char *name = list, *url = list; + + if (list == default_remote) + list = NULL; + + if (!list || !strchr(list, '/')) { + struct remote *remote; + + /* Look up remote */ + remote = remote_get(list); + if (!remote) { + error("no such remote: '%s'", name); + free(list); + return 1; + } + if (!remote->url) { + free(list); + return error(_("remote '%s' has no URLs"), + name); + } + url = remote->url[0]; + } + res = supports_gvfs_protocol(url, NULL); + free(list); + } else if (set) { + res = set_config("gvfs.cache-server=%s", set); + free(set); + } else { + char *url = NULL; + + printf("Using cache server: %s\n", + git_config_get_string("gvfs.cache-server", &url) ? + "(undefined)" : url); + free(url); + } + + return !!res; +} + static struct { const char *name; int (*fn)(int, const char **); @@ -1211,6 +1310,7 @@ static struct { { "help", cmd_help }, { "version", cmd_version }, { "diagnose", cmd_diagnose }, + { "cache-server", cmd_cache_server }, { NULL, NULL}, }; From b03bade0489bd84a0a5d466aa89ed33886be2b36 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 12 May 2021 17:59:58 +0200 Subject: [PATCH 26/35] scalar: add a test toggle to skip accessing the vsts/info endpoint In Scalar's functional tests, we do not do anything with authentication. Therefore, we do want to avoid accessing the `vsts/info` endpoint because it requires authentication even on otherwise public repositories. Let's introduce the environment variable `SCALAR_TEST_SKIP_VSTS_INFO` which can be set to `true` to simply skip that step (and force the `url_*` style repository IDs instead of `id_*` whenever possible). Signed-off-by: Johannes Schindelin --- scalar.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scalar.c b/scalar.c index cd625c3564a120..acb8e5439f425b 100644 --- a/scalar.c +++ b/scalar.c @@ -501,7 +501,8 @@ static char *get_cache_key(const char *url) * The GVFS protocol is only supported via https://; For testing, we * also allow http://. */ - if (can_url_support_gvfs(url)) { + if (!git_env_bool("SCALAR_TEST_SKIP_VSTS_INFO", 0) && + can_url_support_gvfs(url)) { cp.git_cmd = 1; strvec_pushl(&cp.args, "gvfs-helper", "--remote", url, "endpoint", "vsts/info", NULL); From ef8e7df5d23d0ccd64b6192e05b3db2eaae7d282 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 25 Jan 2022 23:49:21 +0100 Subject: [PATCH 27/35] scalar: adjust documentation to the microsoft/git fork Scalar in Microsoft's Git fork can do a little more than Scalar in upstream Git: in Microsoft's Git, it supports the GVFS protocol so that Scalar can clone from Azure DevOps. Signed-off-by: Johannes Schindelin --- Documentation/scalar/getting-started.md | 24 +++++++++++++++--- Documentation/scalar/index.md | 14 +++++++---- Documentation/scalar/philosophy.md | 33 ++++++++++++++----------- Documentation/scalar/troubleshooting.md | 20 +++++++++++++++ 4 files changed, 68 insertions(+), 23 deletions(-) diff --git a/Documentation/scalar/getting-started.md b/Documentation/scalar/getting-started.md index dec60ce407713a..67d20a1f4005c6 100644 --- a/Documentation/scalar/getting-started.md +++ b/Documentation/scalar/getting-started.md @@ -18,8 +18,9 @@ Creating a new Scalar clone --------------------------------------------------- The `clone` verb creates a local enlistment of a remote repository using the -partial clone feature available e.g. on GitHub. - +partial clone feature available e.g. on GitHub, or using the +[GVFS protocol](https://github.com/microsoft/VFSForGit/blob/HEAD/Protocol.md), +such as Azure Repos. ``` scalar clone [options] [] @@ -68,11 +69,26 @@ in ``. These options allow a user to customize their initial enlistment. * `--full-clone`: If specified, do not initialize the sparse-checkout feature. - All files will be present in your `src` directory. This uses a Git partial - clone: blobs are downloaded on demand. + All files will be present in your `src` directory. This behaves very similar + to a Git partial clone in that blobs are downloaded on demand. However, it + will use the GVFS protocol to download all Git objects. + +* `--cache-server-url=`: If specified, set the intended cache server to + the specified ``. All object queries will use the GVFS protocol to this + `` instead of the origin remote. If the remote supplies a list of + cache servers via the `/gvfs/config` endpoint, then the `clone` command + will select a nearby cache server from that list. * `--branch=`: Specify the branch to checkout after clone. +* `--local-cache-path=`: Use this option to override the path for the + local Scalar cache. If not specified, then Scalar will select a default + path to share objects with your other enlistments. On Windows, this path + is a subdirectory of `:\.scalarCache\`. On Mac, this path is a + subdirectory of `~/.scalarCache/`. The default cache path is recommended so + multiple enlistments of the same remote repository share objects on the + same device. + ### Advanced Options The options below are not intended for use by a typical user. These are diff --git a/Documentation/scalar/index.md b/Documentation/scalar/index.md index f9f5ab06e09253..4f56e2b0ebbac6 100644 --- a/Documentation/scalar/index.md +++ b/Documentation/scalar/index.md @@ -28,10 +28,14 @@ these features for that repo (except partial clone) and start running suggested maintenance in the background using [the `git maintenance` feature](https://git-scm.com/docs/git-maintenance). -Repos cloned with the `scalar clone` command use partial clone to significantly -reduce the amount of data required to get started using a repository. By -delaying all blob downloads until they are required, Scalar allows you to work -with very large repositories quickly. +Repos cloned with the `scalar clone` command use partial clone or the +[GVFS protocol](https://github.com/microsoft/VFSForGit/blob/HEAD/Protocol.md) +to significantly reduce the amount of data required to get started +using a repository. By delaying all blob downloads until they are required, +Scalar allows you to work with very large repositories quickly. The GVFS +protocol allows a network of _cache servers_ to serve objects with lower +latency and higher throughput. The cache servers also reduce load on the +central server. Documentation ------------- @@ -42,7 +46,7 @@ Documentation * [Troubleshooting](troubleshooting.md): Collect diagnostic information or update custom settings. Includes - `scalar diagnose`. + `scalar diagnose` and `scalar cache-server`. * [The Philosophy of Scalar](philosophy.md): Why does Scalar work the way it does, and how do we make decisions about its future? diff --git a/Documentation/scalar/philosophy.md b/Documentation/scalar/philosophy.md index 51486a75e41f0d..e3dfa025a2504c 100644 --- a/Documentation/scalar/philosophy.md +++ b/Documentation/scalar/philosophy.md @@ -13,22 +13,27 @@ Scalar only to configure those new settings. In particular, we ported features like background maintenance to Git to make Scalar simpler and make Git more powerful. -Services such as GitHub support partial clone , a standard adopted by the Git -project to download only part of the Git objects when cloning, and fetching -further objects on demand. If your hosting service supports partial clone, then -we absolutely recommend it as a way to greatly speed up your clone and fetch -times and to reduce how much disk space your Git repository requires. Scalar -will help with this! +Scalar ships inside [a custom version of Git][microsoft-git], but we are +working to make it available in other forks of Git. The only feature +that is not intended to ever reach the standard Git client is Scalar's use +of [the GVFS Protocol][gvfs-protocol], which is essentially an older +version of [Git's partial clone feature](https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/) +that was available first in Azure Repos. Services such as GitHub support +only partial clone instead of the GVFS protocol because that is the +standard adopted by the Git project. If your hosting service supports +partial clone, then we absolutely recommend it as a way to greatly speed +up your clone and fetch times and to reduce how much disk space your Git +repository requires. Scalar will help with this! -Most of the value of Scalar can be found in the core Git client. However, most -of the advanced features that really optimize Git's performance are off by -default for compatibility reasons. To really take advantage of Git's latest and -greatest features, you either need to study the [`git config` -documentation](https://git-scm.com/docs/git-config) and regularly read [the Git -release notes](https://github.com/git/git/tree/master/Documentation/RelNotes). +If you don't use the GVFS Protocol, then most of the value of Scalar can +be found in the core Git client. However, most of the advanced features +that really optimize Git's performance are off by default for compatibility +reasons. To really take advantage of Git's latest and greatest features, +you either need to study the [`git config` documentation](https://git-scm.com/docs/git-config) +and regularly read [the Git release notes](https://github.com/git/git/tree/master/Documentation/RelNotes). Even if you do all that work and customize your Git settings on your machines, -you likely will want to share those settings with other team members. Or, you -can just use Scalar! +you likely will want to share those settings with other team members. +Or, you can just use Scalar! Using `scalar register` on an existing Git repository will give you these benefits: diff --git a/Documentation/scalar/troubleshooting.md b/Documentation/scalar/troubleshooting.md index 8ec56ad437ff09..c54d2438f22523 100644 --- a/Documentation/scalar/troubleshooting.md +++ b/Documentation/scalar/troubleshooting.md @@ -18,3 +18,23 @@ files for that repository. This includes: As the `diagnose` command completes, it provides the path of the resulting zip file. This zip can be attached to bug reports to make the analysis easier. + +Modifying Configuration Values +------------------------------ + +The Scalar-specific configuration is only available for repos using the +GVFS protocol. + +### Cache Server URL + +When using an enlistment cloned with `scalar clone` and the GVFS protocol, +you will have a value called the cache server URL. Cache servers are a feature +of the GVFS protocol to provide low-latency access to the on-demand object +requests. This modifies the `gvfs.cache-server` setting in your local Git config +file. + +Run `scalar cache-server --get` to see the current cache server. + +Run `scalar cache-server --list` to see the available cache server URLs. + +Run `scalar cache-server --set=` to set your cache server to ``. From 396e2beec95e60c97dd5f04d7e81d59553d22d75 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 21 Jun 2021 08:21:48 -0400 Subject: [PATCH 28/35] scalar: enable untracked cache unconditionally Signed-off-by: Derrick Stolee --- scalar.c | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/scalar.c b/scalar.c index acb8e5439f425b..dc81af9e05b7ec 100644 --- a/scalar.c +++ b/scalar.c @@ -151,23 +151,7 @@ static int set_recommended_config(int reconfigure) { "core.FSCache", "true", 1 }, { "core.multiPackIndex", "true", 1 }, { "core.preloadIndex", "true", 1 }, -#ifndef WIN32 { "core.untrackedCache", "true", 1 }, -#else - /* - * Unfortunately, Scalar's Functional Tests demonstrated - * that the untracked cache feature is unreliable on Windows - * (which is a bummer because that platform would benefit the - * most from it). For some reason, freshly created files seem - * not to update the directory's `lastModified` time - * immediately, but the untracked cache would need to rely on - * that. - * - * Therefore, with a sad heart, we disable this very useful - * feature on Windows. - */ - { "core.untrackedCache", "false", 1 }, -#endif { "core.logAllRefUpdates", "true", 1 }, { "credential.https://dev.azure.com.useHttpPath", "true", 1 }, { "credential.validate", "false", 1 }, /* GCM4W-only */ From a499f1b9e22f0b81dc7180f1da4f2f160023c332 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 May 2022 00:29:55 +0200 Subject: [PATCH 29/35] scalar diagnose: accommodate Scalar's Functional Tests Those tests specifically verify that the `.zip` file path is shown on `stdout`. Let's do that again, under the assumption that there are scripts out there that rely on this behavior. Signed-off-by: Johannes Schindelin --- diagnose.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/diagnose.c b/diagnose.c index 659792473b4239..e770189b14ece3 100644 --- a/diagnose.c +++ b/diagnose.c @@ -327,10 +327,13 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode) goto diagnose_cleanup; } - fprintf(stderr, "\n" - "Diagnostics complete.\n" - "All of the gathered info is captured in '%s'\n", - zip_path->buf); + strbuf_reset(&buf); + strbuf_addf(&buf, "\n" + "Diagnostics complete.\n" + "All of the gathered info is captured in '%s'\n", + zip_path->buf); + write_or_die(stdout_fd, buf.buf, buf.len); + write_or_die(2, buf.buf, buf.len); diagnose_cleanup: if (archiver_fd >= 0) { From 1756efd84875b8bec817be684e21db0e2c2316d6 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 24 May 2022 14:39:29 -0400 Subject: [PATCH 30/35] setup: add discover_git_directory_reason() There are many reasons why discovering a Git directory may fail. In particular, 8959555cee7 (setup_git_directory(): add an owner check for the top-level directory, 2022-03-02) added ownership checks as a security precaution. Callers attempting to set up a Git directory may want to inform the user about the reason for the failure. For that, expose the enum discovery_result from within setup.c and into cache.h where discover_git_directory() is defined. I initially wanted to change the return type of discover_git_directory() to be this enum, but several callers rely upon the "zero means success". The two problems with this are: 1. The zero value of the enum is actually GIT_DIR_NONE, so nonpositive results are errors. 2. There are multiple successful states, so some positive results are successful. Instead of updating all callers immediately, add a new method, discover_git_directory_reason(), and convert discover_git_directory() to be a thin shim on top of it. Because there are extra checks that discover_git_directory_reason() does after setup_git_directory_gently_1(), there are other modes that can be returned for failure states. Add these modes to the enum, but be sure to explicitly add them as BUG() states in the switch of setup_git_directory_gently(). Signed-off-by: Derrick Stolee --- cache.h | 32 ++++++++++++++++++++++++++++++-- setup.c | 32 +++++++++++--------------------- 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/cache.h b/cache.h index 265a2565e3b5c7..94d4726d424e3b 100644 --- a/cache.h +++ b/cache.h @@ -631,6 +631,30 @@ void set_git_work_tree(const char *tree); #define ALTERNATE_DB_ENVIRONMENT "GIT_ALTERNATE_OBJECT_DIRECTORIES" void setup_work_tree(void); + +/* + * discover_git_directory_reason() is similar to discover_git_directory(), + * except it returns an enum value instead. It is important to note that + * a zero-valued return here is actually GIT_DIR_NONE, which is different + * from discover_git_directory. + */ +enum discovery_result { + GIT_DIR_NONE = 0, + GIT_DIR_EXPLICIT, + GIT_DIR_DISCOVERED, + GIT_DIR_BARE, + /* these are errors */ + GIT_DIR_HIT_CEILING = -1, + GIT_DIR_HIT_MOUNT_POINT = -2, + GIT_DIR_INVALID_GITFILE = -3, + GIT_DIR_INVALID_OWNERSHIP = -4, + GIT_DIR_DISALLOWED_BARE = -5, + GIT_DIR_INVALID_FORMAT = -6, + GIT_DIR_CWD_FAILURE = -7, +}; +enum discovery_result discover_git_directory_reason(struct strbuf *commondir, + struct strbuf *gitdir); + /* * Find the commondir and gitdir of the repository that contains the current * working directory, without changing the working directory or other global @@ -639,8 +663,12 @@ void setup_work_tree(void); * both have the same result appended to the buffer. The return value is * either 0 upon success and non-zero if no repository was found. */ -int discover_git_directory(struct strbuf *commondir, - struct strbuf *gitdir); +static inline int discover_git_directory(struct strbuf *commondir, + struct strbuf *gitdir) +{ + return discover_git_directory_reason(commondir, gitdir) <= 0; +} + const char *setup_git_directory_gently(int *); const char *setup_git_directory(void); char *prefix_path(const char *prefix, int len, const char *path); diff --git a/setup.c b/setup.c index 852b3d9141c6ed..ca3f83cc6d3538 100644 --- a/setup.c +++ b/setup.c @@ -1205,19 +1205,6 @@ static const char *allowed_bare_repo_to_string( return NULL; } -enum discovery_result { - GIT_DIR_NONE = 0, - GIT_DIR_EXPLICIT, - GIT_DIR_DISCOVERED, - GIT_DIR_BARE, - /* these are errors */ - GIT_DIR_HIT_CEILING = -1, - GIT_DIR_HIT_MOUNT_POINT = -2, - GIT_DIR_INVALID_GITFILE = -3, - GIT_DIR_INVALID_OWNERSHIP = -4, - GIT_DIR_DISALLOWED_BARE = -5, -}; - /* * We cannot decide in this function whether we are in the work tree or * not, since the config can only be read _after_ this function was called. @@ -1368,21 +1355,22 @@ static enum discovery_result setup_git_directory_gently_1(struct strbuf *dir, } } -int discover_git_directory(struct strbuf *commondir, - struct strbuf *gitdir) +enum discovery_result discover_git_directory_reason(struct strbuf *commondir, + struct strbuf *gitdir) { struct strbuf dir = STRBUF_INIT, err = STRBUF_INIT; size_t gitdir_offset = gitdir->len, cwd_len; size_t commondir_offset = commondir->len; struct repository_format candidate = REPOSITORY_FORMAT_INIT; + enum discovery_result result; if (strbuf_getcwd(&dir)) - return -1; + return GIT_DIR_CWD_FAILURE; cwd_len = dir.len; - if (setup_git_directory_gently_1(&dir, gitdir, NULL, 0) <= 0) { + if ((result = setup_git_directory_gently_1(&dir, gitdir, NULL, 0)) <= 0) { strbuf_release(&dir); - return -1; + return result; } /* @@ -1412,7 +1400,7 @@ int discover_git_directory(struct strbuf *commondir, strbuf_setlen(commondir, commondir_offset); strbuf_setlen(gitdir, gitdir_offset); clear_repository_format(&candidate); - return -1; + return GIT_DIR_INVALID_FORMAT; } /* take ownership of candidate.partial_clone */ @@ -1421,7 +1409,7 @@ int discover_git_directory(struct strbuf *commondir, candidate.partial_clone = NULL; clear_repository_format(&candidate); - return 0; + return result; } const char *setup_git_directory_gently(int *nongit_ok) @@ -1513,9 +1501,11 @@ const char *setup_git_directory_gently(int *nongit_ok) *nongit_ok = 1; break; case GIT_DIR_NONE: + case GIT_DIR_CWD_FAILURE: + case GIT_DIR_INVALID_FORMAT: /* * As a safeguard against setup_git_directory_gently_1 returning - * this value, fallthrough to BUG. Otherwise it is possible to + * these values, fallthrough to BUG. Otherwise it is possible to * set startup_info->have_repository to 1 when we did nothing to * find a repository. */ From 167dadd5595f4787450d89ef76e46dc6c98a8973 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 24 Aug 2021 18:01:47 +0200 Subject: [PATCH 31/35] scalar: parse `clone --no-fetch-commits-and-trees` for backwards compatibility This option does not do anything anymore, though. Signed-off-by: Johannes Schindelin --- scalar.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scalar.c b/scalar.c index dc81af9e05b7ec..e252669dd53c39 100644 --- a/scalar.c +++ b/scalar.c @@ -665,7 +665,7 @@ static int init_shared_object_cache(const char *url, static int cmd_clone(int argc, const char **argv) { const char *branch = NULL; - int full_clone = 0, single_branch = 0; + int full_clone = 0, single_branch = 0, dummy = 0; const char *cache_server_url = NULL, *local_cache_root = NULL; char *default_cache_server_url = NULL, *local_cache_root_abs = NULL; struct option clone_options[] = { @@ -682,6 +682,8 @@ static int cmd_clone(int argc, const char **argv) OPT_STRING(0, "local-cache-path", &local_cache_root, N_(""), N_("override the path for the local Scalar cache")), + OPT_HIDDEN_BOOL(0, "no-fetch-commits-and-trees", + &dummy, N_("no longer used")), OPT_END(), }; const char * const clone_usage[] = { From 596e540362736f2548a59105e2a3f00c4b391896 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 30 Jun 2022 11:13:34 -0400 Subject: [PATCH 32/35] scalar: put docs back where they are expected Signed-off-by: Derrick Stolee --- {Documentation/scalar => contrib/scalar/docs}/faq.md | 0 {Documentation/scalar => contrib/scalar/docs}/getting-started.md | 0 {Documentation/scalar => contrib/scalar/docs}/index.md | 0 {Documentation/scalar => contrib/scalar/docs}/philosophy.md | 0 {Documentation/scalar => contrib/scalar/docs}/troubleshooting.md | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename {Documentation/scalar => contrib/scalar/docs}/faq.md (100%) rename {Documentation/scalar => contrib/scalar/docs}/getting-started.md (100%) rename {Documentation/scalar => contrib/scalar/docs}/index.md (100%) rename {Documentation/scalar => contrib/scalar/docs}/philosophy.md (100%) rename {Documentation/scalar => contrib/scalar/docs}/troubleshooting.md (100%) diff --git a/Documentation/scalar/faq.md b/contrib/scalar/docs/faq.md similarity index 100% rename from Documentation/scalar/faq.md rename to contrib/scalar/docs/faq.md diff --git a/Documentation/scalar/getting-started.md b/contrib/scalar/docs/getting-started.md similarity index 100% rename from Documentation/scalar/getting-started.md rename to contrib/scalar/docs/getting-started.md diff --git a/Documentation/scalar/index.md b/contrib/scalar/docs/index.md similarity index 100% rename from Documentation/scalar/index.md rename to contrib/scalar/docs/index.md diff --git a/Documentation/scalar/philosophy.md b/contrib/scalar/docs/philosophy.md similarity index 100% rename from Documentation/scalar/philosophy.md rename to contrib/scalar/docs/philosophy.md diff --git a/Documentation/scalar/troubleshooting.md b/contrib/scalar/docs/troubleshooting.md similarity index 100% rename from Documentation/scalar/troubleshooting.md rename to contrib/scalar/docs/troubleshooting.md From 1a2a3662b801ff89692c695c1bdded63829df459 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 8 Jun 2021 09:43:36 +0200 Subject: [PATCH 33/35] ci: run Scalar's Functional Tests Scalar's Functional Test suite is pretty comprehensive, and caught more than just one bug in the built-in FSMonitor that was missed by Git's own test suite. To benefit from this test suite, automatically run it on the `vfs-*` and `features/*` branches. Note: for simplicity, we're building Git from scratch in all matrix jobs. Also note: for speed, we are using `git-sdk-64-minimal`, even if it lacks the `/bin/install` that we need to install Git's files; We're providing a minimal shell script shim instead. Also, we do not need to bother with the Tcl/Tk parts, therefore we're skipping them, too. Finally, we use GIT_FORCE_UNTRACKED_CACHE in the functional tests, to give the untracked cache a thorough work-out. Signed-off-by: Johannes Schindelin --- .github/workflows/scalar-functional-tests.yml | 220 ++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 .github/workflows/scalar-functional-tests.yml diff --git a/.github/workflows/scalar-functional-tests.yml b/.github/workflows/scalar-functional-tests.yml new file mode 100644 index 00000000000000..1defa314429186 --- /dev/null +++ b/.github/workflows/scalar-functional-tests.yml @@ -0,0 +1,220 @@ +name: Scalar Functional Tests + +env: + SCALAR_REPOSITORY: microsoft/scalar + SCALAR_REF: main + DEBUG_WITH_TMATE: false + SCALAR_TEST_SKIP_VSTS_INFO: true + +on: + push: + branches: [ vfs-*, tentative/vfs-* ] + pull_request: + branches: [ vfs-*, features/* ] + +jobs: + scalar: + name: "Scalar Functional Tests" + + strategy: + fail-fast: false + matrix: + # Order by runtime (in descending order) + os: [windows-2019, macos-10.15, ubuntu-18.04, ubuntu-20.04] + # Scalar.NET used to be tested using `features: [false, experimental]` + # But currently, Scalar/C ignores `feature.scalar` altogether, so let's + # save some electrons and run only one of them... + features: [ignored] + exclude: + # The built-in FSMonitor is not (yet) supported on Linux + - os: ubuntu-18.04 + features: experimental + - os: ubuntu-20.04 + features: experimental + runs-on: ${{ matrix.os }} + + env: + BUILD_FRAGMENT: bin/Release/netcoreapp3.1 + GIT_FORCE_UNTRACKED_CACHE: 1 + + steps: + - name: Check out Git's source code + uses: actions/checkout@v2 + + - name: Setup build tools on Windows + if: runner.os == 'Windows' + uses: git-for-windows/setup-git-for-windows-sdk@v1 + + - name: Provide a minimal `install` on Windows + if: runner.os == 'Windows' + shell: bash + run: | + test -x /usr/bin/install || + tr % '\t' >/usr/bin/install <<-\EOF + #!/bin/sh + + cmd=cp + while test $# != 0 + do + %case "$1" in + %-d) cmd="mkdir -p";; + %-m) shift;; # ignore mode + %*) break;; + %esac + %shift + done + + exec $cmd "$@" + EOF + + - name: Install build dependencies for Git (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get -q -y install libssl-dev libcurl4-openssl-dev gettext + + - name: Build and install Git + shell: bash + env: + NO_TCLTK: Yup + run: | + # We do require a VFS version + def_ver="$(sed -n 's/DEF_VER=\(.*vfs.*\)/\1/p' GIT-VERSION-GEN)" + test -n "$def_ver" + + # Ensure that `git version` reflects DEF_VER + case "$(git describe --match "v[0-9]*vfs*" HEAD)" in + ${def_ver%%.vfs.*}.vfs.*) ;; # okay, we can use this + *) git -c user.name=ci -c user.email=ci@github tag -m for-testing ${def_ver}.NNN.g$(git rev-parse --short HEAD);; + esac + + SUDO= + extra= + case "${{ runner.os }}" in + Windows) + extra=DESTDIR=/c/Progra~1/Git + cygpath -aw "/c/Program Files/Git/cmd" >>$GITHUB_PATH + ;; + Linux) + SUDO=sudo + extra=prefix=/usr + ;; + macOS) + SUDO=sudo + extra=prefix=/usr/local + ;; + esac + + $SUDO make -j5 $extra install + + - name: Ensure that we use the built Git and Scalar + shell: bash + run: | + type -p git + git version + case "$(git version)" in *.vfs.*) echo Good;; *) exit 1;; esac + type -p scalar + scalar version + case "$(scalar version 2>&1)" in *.vfs.*) echo Good;; *) exit 1;; esac + + - name: Check out Scalar's source code + uses: actions/checkout@v2 + with: + fetch-depth: 0 # Indicate full history so Nerdbank.GitVersioning works. + path: scalar + repository: ${{ env.SCALAR_REPOSITORY }} + ref: ${{ env.SCALAR_REF }} + + - name: Setup .NET Core + uses: actions/setup-dotnet@v1 + with: + dotnet-version: 3.1.302 + + - name: Install dependencies + run: dotnet restore + working-directory: scalar + env: + DOTNET_NOLOGO: 1 + + - name: Build + working-directory: scalar + run: dotnet build --configuration Release --no-restore -p:UseAppHost=true # Force generation of executable on macOS. + + - name: Setup platform (Linux) + if: runner.os == 'Linux' + run: | + echo "BUILD_PLATFORM=${{ runner.os }}" >>$GITHUB_ENV + echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$GITHUB_ENV + + - name: Setup platform (Mac) + if: runner.os == 'macOS' + run: | + echo 'BUILD_PLATFORM=Mac' >>$GITHUB_ENV + echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$GITHUB_ENV + + - name: Setup platform (Windows) + if: runner.os == 'Windows' + run: | + echo "BUILD_PLATFORM=${{ runner.os }}" >>$env:GITHUB_ENV + echo 'BUILD_FILE_EXT=.exe' >>$env:GITHUB_ENV + echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$env:GITHUB_ENV + + - name: Configure feature.scalar + run: git config --global feature.scalar ${{ matrix.features }} + + - id: functional_test + name: Functional test + timeout-minutes: 60 + working-directory: scalar + shell: bash + run: | + export GIT_TRACE2_EVENT="$PWD/$TRACE2_BASENAME/Event" + export GIT_TRACE2_PERF="$PWD/$TRACE2_BASENAME/Perf" + export GIT_TRACE2_EVENT_BRIEF=true + export GIT_TRACE2_PERF_BRIEF=true + mkdir -p "$TRACE2_BASENAME" + mkdir -p "$TRACE2_BASENAME/Event" + mkdir -p "$TRACE2_BASENAME/Perf" + git version --build-options + cd ../out + Scalar.FunctionalTests/$BUILD_FRAGMENT/Scalar.FunctionalTests$BUILD_FILE_EXT --test-scalar-on-path --test-git-on-path --timeout=300000 --full-suite + + - name: Force-stop FSMonitor daemons and Git processes (Windows) + if: runner.os == 'Windows' && (success() || failure()) + shell: bash + run: | + set -x + wmic process get CommandLine,ExecutablePath,HandleCount,Name,ParentProcessID,ProcessID + wmic process where "CommandLine Like '%fsmonitor--daemon %run'" delete + wmic process where "ExecutablePath Like '%git.exe'" delete + + - id: trace2_zip_unix + if: runner.os != 'Windows' && ( success() || failure() ) && ( steps.functional_test.conclusion == 'success' || steps.functional_test.conclusion == 'failure' ) + name: Zip Trace2 Logs (Unix) + shell: bash + working-directory: scalar + run: zip -q -r $TRACE2_BASENAME.zip $TRACE2_BASENAME/ + + - id: trace2_zip_windows + if: runner.os == 'Windows' && ( success() || failure() ) && ( steps.functional_test.conclusion == 'success' || steps.functional_test.conclusion == 'failure' ) + name: Zip Trace2 Logs (Windows) + working-directory: scalar + run: Compress-Archive -DestinationPath ${{ env.TRACE2_BASENAME }}.zip -Path ${{ env.TRACE2_BASENAME }} + + - name: Archive Trace2 Logs + if: ( success() || failure() ) && ( steps.trace2_zip_unix.conclusion == 'success' || steps.trace2_zip_windows.conclusion == 'success' ) + uses: actions/upload-artifact@v2 + with: + name: ${{ env.TRACE2_BASENAME }}.zip + path: scalar/${{ env.TRACE2_BASENAME }}.zip + retention-days: 3 + + # The GitHub Action `action-tmate` allows developers to connect to the running agent + # using SSH (it will be a `tmux` session; on Windows agents it will be inside the MSYS2 + # environment in `C:\msys64`, therefore it can be slightly tricky to interact with + # Git for Windows, which runs a slightly incompatible MSYS2 runtime). + - name: action-tmate + if: env.DEBUG_WITH_TMATE == 'true' && failure() + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true From 2a0a8d7f0312fa709e7ffb31c1145daa3c75090e Mon Sep 17 00:00:00 2001 From: Victoria Dye Date: Tue, 5 Apr 2022 13:42:23 -0700 Subject: [PATCH 34/35] scalar: upgrade to newest FSMonitor config setting When FSMonitor was upstreamed, the 'core.useBuiltinFSMonitor' config was deprecated and replaced with an overload of the 'core.fsmonitor' config (i.e., if a boolean value was specified in 'core.fsmonitor', it is treated the way 'core.useBuiltinFSMonitor' originally was). Because 'scalar register' actively sets that config, use it to upgrade the deprecated config setting. Co-authored-by: Johannes Schindelin Signed-off-by: Victoria Dye --- scalar.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/scalar.c b/scalar.c index e252669dd53c39..1e8bae6bab5063 100644 --- a/scalar.c +++ b/scalar.c @@ -183,6 +183,23 @@ static int set_recommended_config(int reconfigure) int i; char *value; + /* + * If a user has "core.usebuiltinfsmonitor" enabled, try to switch to + * the new (non-deprecated) setting (core.fsmonitor). + */ + if (!git_config_get_string("core.usebuiltinfsmonitor", &value)) { + char *dummy = NULL; + if (git_config_get_string("core.fsmonitor", &dummy) && + git_config_set_gently("core.fsmonitor", value) < 0) + return error(_("could not configure %s=%s"), + "core.fsmonitor", value); + if (git_config_set_gently("core.usebuiltinfsmonitor", NULL) < 0) + return error(_("could not configure %s=%s"), + "core.useBuiltinFSMonitor", "NULL"); + free(value); + free(dummy); + } + for (i = 0; config[i].key; i++) { if (set_scalar_config(config + i, reconfigure)) return error(_("could not configure %s=%s"), From db8dd2df4d31d9de0ae1020b64043ebba58e09f1 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 23 May 2022 10:56:35 -0400 Subject: [PATCH 35/35] scalar reconfigure: help users remove buggy repos When running 'scalar reconfigure -a', such as at install time, Scalar has warning messages about the repository missing (or not containing a .git directory). Failures can also happen while trying to modify the repository-local config for that repository. These warnings may seem confusing to users who don't understand what they mean or how to stop them. Add a warning that instructs the user how to remove the warning in future installations. Signed-off-by: Derrick Stolee --- scalar.c | 48 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/scalar.c b/scalar.c index 1e8bae6bab5063..e8f73e53b5a3e9 100644 --- a/scalar.c +++ b/scalar.c @@ -992,6 +992,7 @@ static int cmd_reconfigure(int argc, const char **argv) git_config(get_scalar_repos, &scalar_repos); for (i = 0; i < scalar_repos.nr; i++) { + int failed = 0; const char *dir = scalar_repos.items[i].string; strbuf_reset(&commondir); @@ -999,19 +1000,40 @@ static int cmd_reconfigure(int argc, const char **argv) if (chdir(dir) < 0) { warning_errno(_("could not switch to '%s'"), dir); - res = -1; - } else if (discover_git_directory(&commondir, &gitdir) < 0) { - warning_errno(_("git repository gone in '%s'"), dir); - res = -1; - } else { - git_config_clear(); - - the_repository = &r; - r.commondir = commondir.buf; - r.gitdir = gitdir.buf; - - if (set_recommended_config(1) < 0) - res = -1; + failed = -1; + goto loop_end; + } + + switch (discover_git_directory_reason(&commondir, &gitdir)) { + case GIT_DIR_INVALID_OWNERSHIP: + warning(_("repository at '%s' has different owner"), dir); + failed = -1; + goto loop_end; + + case GIT_DIR_DISCOVERED: + break; + + default: + warning(_("repository not found in '%s'"), dir); + failed = -1; + break; + } + + git_config_clear(); + + the_repository = &r; + r.commondir = commondir.buf; + r.gitdir = gitdir.buf; + + if (set_recommended_config(1) < 0) + failed = -1; + +loop_end: + if (failed) { + res = failed; + warning(_("to unregister this repository from Scalar, run\n" + "\tgit config --global --unset --fixed-value scalar.repo \"%s\""), + dir); } }