Integrate Scalar (ported to C) into vfs-2.32.0

.github/workflows/main.yml

@@ -4,6 +4,7 @@ on: [push, pull_request]
 
 env:
   DEVELOPER: 1
+  INCLUDE_SCALAR: YesPlease
 
 jobs:
   ci-config:

.github/workflows/scalar-functional-tests.yml

@@ -1,9 +1,10 @@
 name: Scalar Functional Tests
 
 env:
-  SCALAR_REPOSITORY: dscho/scalar
-  SCALAR_REF: vfs-2.32.0
+  SCALAR_REPOSITORY: derrickstolee/scalar
+  SCALAR_REF: test-scalar-in-c
   DEBUG_WITH_TMATE: false
+  SCALAR_TEST_SKIP_VSTS_INFO: true
 
 on:
   push:
@@ -20,7 +21,10 @@ jobs:
       matrix:
         # Order by runtime (in descending order)
         os: [windows-2019, macos-10.15, ubuntu-16.04, ubuntu-18.04, ubuntu-20.04]
-        features: [false, experimental]
+        # Scalar.NET used to be tested using `features: [false, experimental]`
+        # But currently, Scalar/C ignores `feature.scalar` altogether, so let's
+        # save some electrons and run only one of them...
+        features: [ignored]
         exclude:
           # The built-in FSMonitor is not (yet) supported on Linux
           - os: ubuntu-16.04
@@ -102,15 +106,17 @@ jobs:
             ;;
           esac
 
-          $SUDO make -j5 $extra install
+          $SUDO make -j5 INCLUDE_SCALAR=AbsolutelyYes $extra install
 
-      - name: Ensure that we use the built Git (Windows)
-        if: runner.os == 'Windows'
-        shell: powershell
+      - name: Ensure that we use the built Git and Scalar
+        shell: bash
         run: |
-          cmd /c where git
+          type -p git
           git version
-          if ((git version) -like "*.vfs.*") { echo Good } else { exit 1 }
+          case "$(git version)" in *.vfs.*) echo Good;; *) exit 1;; esac
+          type -p scalar
+          scalar version
+          case "$(scalar version 2>&1)" in *.vfs.*) echo Good;; *) exit 1;; esac
 
       - name: Check out Scalar's source code
         uses: actions/checkout@v2
@@ -172,11 +178,10 @@ jobs:
           mkdir -p "$TRACE2_BASENAME/Perf"
           git version --build-options
           cd ../out
-          PATH="$PWD/Scalar/$BUILD_FRAGMENT:$PWD/Scalar.Service/$BUILD_FRAGMENT:$PATH"
           Scalar.FunctionalTests/$BUILD_FRAGMENT/Scalar.FunctionalTests$BUILD_FILE_EXT --test-scalar-on-path --test-git-on-path --timeout=300000 --full-suite
 
       - name: Force-stop FSMonitor daemons and Git processes (Windows)
-        if: runner.os == 'Windows' && matrix.features == 'experimental' && (success() || failure())
+        if: runner.os == 'Windows' && (success() || failure())
         shell: bash
         run: |
           set -x

Documentation/config/core.txt

@@ -726,3 +726,12 @@ core.abbrev::
 	If set to "no", no abbreviation is made and the object names
 	are shown in their full length.
 	The minimum length is 4.
+
+core.configWriteLockTimeoutMS::
+	When processes try to write to the config concurrently, it is likely
+	that one process "wins" and the other process(es) fail to lock the
+	config file. By configuring a timeout larger than zero, Git can be
+	told to try to lock the config again a couple times within the
+	specified timeout. If the timeout is configure to zero (which is the
+	default), Git will fail immediately when the config is already
+	locked.

Makefile

@@ -1966,6 +1966,10 @@ ifndef PAGER_ENV
 PAGER_ENV = LESS=FRX LV=-c
 endif
 
+ifneq (,$(INCLUDE_SCALAR))
+EXTRA_PROGRAMS += contrib/scalar/scalar$X
+endif
+
 QUIET_SUBDIR0  = +$(MAKE) -C # space to separate -C and subdir
 QUIET_SUBDIR1  =
 
@@ -2471,6 +2475,10 @@ endif
 .PHONY: objects
 objects: $(OBJECTS)
 
+SCALAR_SOURCES := contrib/scalar/scalar.c contrib/scalar/json-parser.c
+SCALAR_OBJECTS := $(SCALAR_SOURCES:c=o)
+OBJECTS += $(SCALAR_OBJECTS)
+
 dep_files := $(foreach f,$(OBJECTS),$(dir $f).depend/$(notdir $f).d)
 dep_dirs := $(addsuffix .depend,$(sort $(dir $(OBJECTS))))
 
@@ -2617,6 +2625,13 @@ $(REMOTE_CURL_PRIMARY): remote-curl.o http.o http-walker.o GIT-LDFLAGS $(GITLIBS
 	$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \
 		$(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS)
 
+contrib/scalar/scalar$X: $(SCALAR_OBJECTS) GIT-LDFLAGS $(GITLIBS)
+	$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) \
+		$(filter %.o,$^) $(LIBS)
+
+bin-wrappers/scalar: contrib/scalar/Makefile
+	$(QUIET_SUBDIR0)contrib/scalar $(QUIET_SUBDIR1) ../../bin-wrappers/scalar
+
 git-gvfs-helper$X: gvfs-helper.o http.o GIT-LDFLAGS $(GITLIBS)
 	$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \
 		$(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS)
@@ -2640,14 +2655,23 @@ Documentation/GIT-EXCLUDED-PROGRAMS: FORCE
 .PHONY: doc man man-perl html info pdf
 doc: man-perl
 	$(MAKE) -C Documentation all
+ifneq (,$(INCLUDE_SCALAR))
+	$(QUIET_SUBDIR0)contrib/scalar $(QUIET_SUBDIR1) scalar.html scalar.1
+endif
 
 man: man-perl
 	$(MAKE) -C Documentation man
+ifneq (,$(INCLUDE_SCALAR))
+	$(QUIET_SUBDIR0)contrib/scalar $(QUIET_SUBDIR1) scalar.1
+endif
 
 man-perl: perl/build/man/man3/Git.3pm
 
 html:
 	$(MAKE) -C Documentation html
+ifneq (,$(INCLUDE_SCALAR))
+	$(QUIET_SUBDIR0)contrib/scalar $(QUIET_SUBDIR1) scalar.html
+endif
 
 info:
 	$(MAKE) -C Documentation info
@@ -2886,6 +2910,10 @@ endif
 
 test_bindir_programs := $(patsubst %,bin-wrappers/%,$(BINDIR_PROGRAMS_NEED_X) $(BINDIR_PROGRAMS_NO_X) $(TEST_PROGRAMS_NEED_X))
 
+ifneq (,$(INCLUDE_SCALAR))
+test_bindir_programs += bin-wrappers/scalar
+endif
+
 all:: $(TEST_PROGRAMS) $(test_bindir_programs)
 
 bin-wrappers/%: wrap-for-bin.sh
@@ -2906,6 +2934,9 @@ export TEST_NO_MALLOC_CHECK
 
 test: all
 	$(MAKE) -C t/ all
+ifneq (,$(INCLUDE_SCALAR))
+	$(MAKE) -C contrib/scalar/t
+endif
 
 perf: all
 	$(MAKE) -C t/perf/ all
@@ -3031,6 +3062,9 @@ install: all
 	$(INSTALL) $(ALL_PROGRAMS) '$(DESTDIR_SQ)$(gitexec_instdir_SQ)'
 	$(INSTALL) -m 644 $(SCRIPT_LIB) '$(DESTDIR_SQ)$(gitexec_instdir_SQ)'
 	$(INSTALL) $(install_bindir_programs) '$(DESTDIR_SQ)$(bindir_SQ)'
+ifneq (,$(INCLUDE_SCALAR))
+	$(INSTALL) contrib/scalar/scalar$X '$(DESTDIR_SQ)$(bindir_SQ)'
+endif
 ifdef MSVC
 	# We DO NOT install the individual foo.o.pdb files because they
 	# have already been rolled up into the exe's pdb file.
@@ -3124,6 +3158,10 @@ install-doc: install-man-perl
 
 install-man: install-man-perl
 	$(MAKE) -C Documentation install-man
+ifneq (,$(INCLUDE_SCALAR))
+	$(MAKE) -C contrib/scalar scalar.1
+	$(INSTALL) contrib/scalar/scalar.1 '$(DESTDIR_SQ)$(mandir_SQ)/man1'
+endif
 
 install-man-perl: man-perl
 	$(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(mandir_SQ)/man3'
@@ -3132,6 +3170,10 @@ install-man-perl: man-perl
 
 install-html:
 	$(MAKE) -C Documentation install-html
+ifneq (,$(INCLUDE_SCALAR))
+	$(MAKE) -C contrib/scalar scalar.html
+	$(INSTALL) contrib/scalar/scalar.html '$(DESTDIR_SQ)$(htmldir)'
+endif
 
 install-info:
 	$(MAKE) -C Documentation install-info
@@ -3269,6 +3311,9 @@ endif
 ifndef NO_TCLTK
 	$(MAKE) -C gitk-git clean
 	$(MAKE) -C git-gui clean
+endif
+ifneq (,$(INCLUDE_SCALAR))
+	$(MAKE) -C contrib/scalar clean
 endif
 	$(RM) GIT-VERSION-FILE GIT-CFLAGS GIT-LDFLAGS GIT-BUILD-OPTIONS
 	$(RM) GIT-USER-AGENT GIT-PREFIX

builtin/gc.c

@@ -1619,18 +1619,40 @@ static int launchctl_remove_plists(const char *cmd)
 		launchctl_remove_plist(SCHEDULE_WEEKLY, cmd);
 }
 
+static int launchctl_list_contains_plist(const char *name, const char *cmd)
+{
+	int result;
+	struct child_process child = CHILD_PROCESS_INIT;
+	char *uid = launchctl_get_uid();
+
+	strvec_split(&child.args, cmd);
+	strvec_pushl(&child.args, "list", name, NULL);
+
+	child.no_stderr = 1;
+	child.no_stdout = 1;
+
+	if (start_command(&child))
+		die(_("failed to start launchctl"));
+
+	result = finish_command(&child);
+
+	free(uid);
+
+	/* Returns failure if 'name' doesn't exist. */
+	return !result;
+}
+
 static int launchctl_schedule_plist(const char *exec_path, enum schedule_priority schedule, const char *cmd)
 {
-	FILE *plist;
-	int i;
+	int i, fd;
 	const char *preamble, *repeat;
 	const char *frequency = get_frequency(schedule);
 	char *name = launchctl_service_name(frequency);
 	char *filename = launchctl_service_filename(name);
-
-	if (safe_create_leading_directories(filename))
-		die(_("failed to create directories for '%s'"), filename);
-	plist = xfopen(filename, "w");
+	struct lock_file lk = LOCK_INIT;
+	static unsigned long lock_file_timeout_ms = ULONG_MAX;
+	struct strbuf plist = STRBUF_INIT, plist2 = STRBUF_INIT;
+	struct stat st;
 
 	preamble = "<?xml version=\"1.0\"?>\n"
 		   "<!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">\n"
@@ -1649,7 +1671,7 @@ static int launchctl_schedule_plist(const char *exec_path, enum schedule_priorit
 		   "</array>\n"
 		   "<key>StartCalendarInterval</key>\n"
 		   "<array>\n";
-	fprintf(plist, preamble, name, exec_path, exec_path, frequency);
+	strbuf_addf(&plist, preamble, name, exec_path, exec_path, frequency);
 
 	switch (schedule) {
 	case SCHEDULE_HOURLY:
@@ -1658,7 +1680,7 @@ static int launchctl_schedule_plist(const char *exec_path, enum schedule_priorit
 			 "<key>Minute</key><integer>0</integer>\n"
 			 "</dict>\n";
 		for (i = 1; i <= 23; i++)
-			fprintf(plist, repeat, i);
+			strbuf_addf(&plist, repeat, i);
 		break;
 
 	case SCHEDULE_DAILY:
@@ -1668,32 +1690,59 @@ static int launchctl_schedule_plist(const char *exec_path, enum schedule_priorit
 			 "<key>Minute</key><integer>0</integer>\n"
 			 "</dict>\n";
 		for (i = 1; i <= 6; i++)
-			fprintf(plist, repeat, i);
+			strbuf_addf(&plist, repeat, i);
 		break;
 
 	case SCHEDULE_WEEKLY:
-		fprintf(plist,
-			"<dict>\n"
-			"<key>Day</key><integer>0</integer>\n"
-			"<key>Hour</key><integer>0</integer>\n"
-			"<key>Minute</key><integer>0</integer>\n"
-			"</dict>\n");
+		strbuf_addstr(&plist,
+			      "<dict>\n"
+			      "<key>Day</key><integer>0</integer>\n"
+			      "<key>Hour</key><integer>0</integer>\n"
+			      "<key>Minute</key><integer>0</integer>\n"
+			      "</dict>\n");
 		break;
 
 	default:
 		/* unreachable */
 		break;
 	}
-	fprintf(plist, "</array>\n</dict>\n</plist>\n");
-	fclose(plist);
+	strbuf_addstr(&plist, "</array>\n</dict>\n</plist>\n");
+
+	if (safe_create_leading_directories(filename))
+		die(_("failed to create directories for '%s'"), filename);
+
+	if ((long)lock_file_timeout_ms < 0 &&
+	    git_config_get_ulong("gc.launchctlplistlocktimeoutms",
+				 &lock_file_timeout_ms))
+		lock_file_timeout_ms = 150;
+
+	fd = hold_lock_file_for_update_timeout(&lk, filename, LOCK_DIE_ON_ERROR,
+					       lock_file_timeout_ms);
 
-	/* bootout might fail if not already running, so ignore */
-	launchctl_boot_plist(0, filename, cmd);
-	if (launchctl_boot_plist(1, filename, cmd))
-		die(_("failed to bootstrap service %s"), filename);
+	/*
+	 * Does this file already exist? With the intended contents? Is it
+	 * registered already? Then it does not need to be re-registered.
+	 */
+	if (!stat(filename, &st) && st.st_size == plist.len &&
+	    strbuf_read_file(&plist2, filename, plist.len) == plist.len &&
+	    !strbuf_cmp(&plist, &plist2) &&
+	    launchctl_list_contains_plist(name, cmd))
+		rollback_lock_file(&lk);
+	else {
+		if (write_in_full(fd, plist.buf, plist.len) < 0 ||
+		    commit_lock_file(&lk))
+			die_errno(_("could not write '%s'"), filename);
+
+		/* bootout might fail if not already running, so ignore */
+		launchctl_boot_plist(0, filename, cmd);
+		if (launchctl_boot_plist(1, filename, cmd))
+			die(_("failed to bootstrap service %s"), filename);
+	}
 
 	free(filename);
 	free(name);
+	strbuf_release(&plist);
+	strbuf_release(&plist2);
 	return 0;
 }
 

builtin/help.c

@@ -395,6 +395,8 @@ static const char *cmd_to_page(const char *git_cmd)
 		return git_cmd;
 	else if (is_git_command(git_cmd))
 		return xstrfmt("git-%s", git_cmd);
+	else if (!strcmp("scalar", git_cmd))
+		return xstrdup(git_cmd);
 	else
 		return xstrfmt("git%s", git_cmd);
 }

ci/run-test-slice.sh

@@ -14,4 +14,9 @@ make --quiet -C t T="$(cd t &&
 	./helper/test-tool path-utils slice-tests "$1" "$2" t[0-9]*.sh |
 	tr '\n' ' ')"
 
+if test 0 = "$1" && test -n "$INCLUDE_SCALAR"
+then
+	make -C contrib/scalar/t
+fi
+
 check_unignored_build_artifacts

config.c

@@ -3024,6 +3024,7 @@ int git_config_set_multivar_in_file_gently(const char *config_filename,
 					   const char *value_pattern,
 					   unsigned flags)
 {
+	static unsigned long timeout_ms = ULONG_MAX;
 	int fd = -1, in_fd = -1;
 	int ret;
 	struct lock_file lock = LOCK_INIT;
@@ -3044,11 +3045,16 @@ int git_config_set_multivar_in_file_gently(const char *config_filename,
 	if (!config_filename)
 		config_filename = filename_buf = git_pathdup("config");
 
+	if ((long)timeout_ms < 0 &&
+	    git_config_get_ulong("core.configWriteLockTimeoutMS", &timeout_ms))
+		timeout_ms = 0;
+
 	/*
 	 * The lock serves a purpose in addition to locking: the new
 	 * contents of .git/config will be written into it.
 	 */
-	fd = hold_lock_file_for_update(&lock, config_filename, 0);
+	fd = hold_lock_file_for_update_timeout(&lock, config_filename, 0,
+					       timeout_ms);
 	if (fd < 0) {
 		error_errno(_("could not lock config file %s"), config_filename);
 		ret = CONFIG_NO_LOCK;