Skip to content

Commit

Permalink
Merge pull request #371 from dscho/run-scalar-functional-tests-and-fi…
Browse files Browse the repository at this point in the history
…x-built-in-fsmonitor

Fix the built-in FSMonitor, and run Scalar's Functional Tests as part of the automated builds
  • Loading branch information
dscho committed Nov 3, 2023
2 parents 9a288c6 + 6875b09 commit 76b3495
Show file tree
Hide file tree
Showing 23 changed files with 1,778 additions and 84 deletions.
220 changes: 220 additions & 0 deletions .github/workflows/scalar-functional-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,220 @@
name: Scalar Functional Tests

env:
SCALAR_REPOSITORY: microsoft/scalar
SCALAR_REF: main
DEBUG_WITH_TMATE: false
SCALAR_TEST_SKIP_VSTS_INFO: true

on:
push:
branches: [ vfs-*, tentative/vfs-* ]
pull_request:
branches: [ vfs-*, features/* ]

jobs:
scalar:
name: "Scalar Functional Tests"

strategy:
fail-fast: false
matrix:
# Order by runtime (in descending order)
os: [windows-2019, macos-11, ubuntu-20.04, ubuntu-22.04]
# Scalar.NET used to be tested using `features: [false, experimental]`
# But currently, Scalar/C ignores `feature.scalar` altogether, so let's
# save some electrons and run only one of them...
features: [ignored]
exclude:
# The built-in FSMonitor is not (yet) supported on Linux
- os: ubuntu-20.04
features: experimental
- os: ubuntu-22.04
features: experimental
runs-on: ${{ matrix.os }}

env:
BUILD_FRAGMENT: bin/Release/netcoreapp3.1
GIT_FORCE_UNTRACKED_CACHE: 1

steps:
- name: Check out Git's source code
uses: actions/checkout@v3

- name: Setup build tools on Windows
if: runner.os == 'Windows'
uses: git-for-windows/setup-git-for-windows-sdk@v1

- name: Provide a minimal `install` on Windows
if: runner.os == 'Windows'
shell: bash
run: |
test -x /usr/bin/install ||
tr % '\t' >/usr/bin/install <<-\EOF
#!/bin/sh
cmd=cp
while test $# != 0
do
%case "$1" in
%-d) cmd="mkdir -p";;
%-m) shift;; # ignore mode
%*) break;;
%esac
%shift
done
exec $cmd "$@"
EOF
- name: Install build dependencies for Git (Linux)
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get -q -y install libssl-dev libcurl4-openssl-dev gettext
- name: Build and install Git
shell: bash
env:
NO_TCLTK: Yup
run: |
# We do require a VFS version
def_ver="$(sed -n 's/DEF_VER=\(.*vfs.*\)/\1/p' GIT-VERSION-GEN)"
test -n "$def_ver"
# Ensure that `git version` reflects DEF_VER
case "$(git describe --match "v[0-9]*vfs*" HEAD)" in
${def_ver%%.vfs.*}.vfs.*) ;; # okay, we can use this
*) git -c user.name=ci -c user.email=ci@github tag -m for-testing ${def_ver}.NNN.g$(git rev-parse --short HEAD);;
esac
SUDO=
extra=
case "${{ runner.os }}" in
Windows)
extra=DESTDIR=/c/Progra~1/Git
cygpath -aw "/c/Program Files/Git/cmd" >>$GITHUB_PATH
;;
Linux)
SUDO=sudo
extra=prefix=/usr
;;
macOS)
SUDO=sudo
extra=prefix=/usr/local
;;
esac
$SUDO make -j5 $extra install
- name: Ensure that we use the built Git and Scalar
shell: bash
run: |
type -p git
git version
case "$(git version)" in *.vfs.*) echo Good;; *) exit 1;; esac
type -p scalar
scalar version
case "$(scalar version 2>&1)" in *.vfs.*) echo Good;; *) exit 1;; esac
- name: Check out Scalar's source code
uses: actions/checkout@v3
with:
fetch-depth: 0 # Indicate full history so Nerdbank.GitVersioning works.
path: scalar
repository: ${{ env.SCALAR_REPOSITORY }}
ref: ${{ env.SCALAR_REF }}

- name: Setup .NET Core
uses: actions/setup-dotnet@v3
with:
dotnet-version: '3.1.x'

- name: Install dependencies
run: dotnet restore
working-directory: scalar
env:
DOTNET_NOLOGO: 1

- name: Build
working-directory: scalar
run: dotnet build --configuration Release --no-restore -p:UseAppHost=true # Force generation of executable on macOS.

- name: Setup platform (Linux)
if: runner.os == 'Linux'
run: |
echo "BUILD_PLATFORM=${{ runner.os }}" >>$GITHUB_ENV
echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$GITHUB_ENV
- name: Setup platform (Mac)
if: runner.os == 'macOS'
run: |
echo 'BUILD_PLATFORM=Mac' >>$GITHUB_ENV
echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$GITHUB_ENV
- name: Setup platform (Windows)
if: runner.os == 'Windows'
run: |
echo "BUILD_PLATFORM=${{ runner.os }}" >>$env:GITHUB_ENV
echo 'BUILD_FILE_EXT=.exe' >>$env:GITHUB_ENV
echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$env:GITHUB_ENV
- name: Configure feature.scalar
run: git config --global feature.scalar ${{ matrix.features }}

- id: functional_test
name: Functional test
timeout-minutes: 60
working-directory: scalar
shell: bash
run: |
export GIT_TRACE2_EVENT="$PWD/$TRACE2_BASENAME/Event"
export GIT_TRACE2_PERF="$PWD/$TRACE2_BASENAME/Perf"
export GIT_TRACE2_EVENT_BRIEF=true
export GIT_TRACE2_PERF_BRIEF=true
mkdir -p "$TRACE2_BASENAME"
mkdir -p "$TRACE2_BASENAME/Event"
mkdir -p "$TRACE2_BASENAME/Perf"
git version --build-options
cd ../out
Scalar.FunctionalTests/$BUILD_FRAGMENT/Scalar.FunctionalTests$BUILD_FILE_EXT --test-scalar-on-path --test-git-on-path --timeout=300000 --full-suite
- name: Force-stop FSMonitor daemons and Git processes (Windows)
if: runner.os == 'Windows' && (success() || failure())
shell: bash
run: |
set -x
wmic process get CommandLine,ExecutablePath,HandleCount,Name,ParentProcessID,ProcessID
wmic process where "CommandLine Like '%fsmonitor--daemon %run'" delete
wmic process where "ExecutablePath Like '%git.exe'" delete
- id: trace2_zip_unix
if: runner.os != 'Windows' && ( success() || failure() ) && ( steps.functional_test.conclusion == 'success' || steps.functional_test.conclusion == 'failure' )
name: Zip Trace2 Logs (Unix)
shell: bash
working-directory: scalar
run: zip -q -r $TRACE2_BASENAME.zip $TRACE2_BASENAME/

- id: trace2_zip_windows
if: runner.os == 'Windows' && ( success() || failure() ) && ( steps.functional_test.conclusion == 'success' || steps.functional_test.conclusion == 'failure' )
name: Zip Trace2 Logs (Windows)
working-directory: scalar
run: Compress-Archive -DestinationPath ${{ env.TRACE2_BASENAME }}.zip -Path ${{ env.TRACE2_BASENAME }}

- name: Archive Trace2 Logs
if: ( success() || failure() ) && ( steps.trace2_zip_unix.conclusion == 'success' || steps.trace2_zip_windows.conclusion == 'success' )
uses: actions/upload-artifact@v3
with:
name: ${{ env.TRACE2_BASENAME }}.zip
path: scalar/${{ env.TRACE2_BASENAME }}.zip
retention-days: 3

# The GitHub Action `action-tmate` allows developers to connect to the running agent
# using SSH (it will be a `tmux` session; on Windows agents it will be inside the MSYS2
# environment in `C:\msys64`, therefore it can be slightly tricky to interact with
# Git for Windows, which runs a slightly incompatible MSYS2 runtime).
- name: action-tmate
if: env.DEBUG_WITH_TMATE == 'true' && failure()
uses: mxschmitt/action-tmate@v3
with:
limit-access-to-actor: true
9 changes: 9 additions & 0 deletions Documentation/config/core.txt
Original file line number Diff line number Diff line change
Expand Up @@ -812,3 +812,12 @@ core.WSLCompat::
The default value is false. When set to true, Git will set the mode
bits of the file in the way of wsl, so that the executable flag of
files can be set or read correctly.

core.configWriteLockTimeoutMS::
When processes try to write to the config concurrently, it is likely
that one process "wins" and the other process(es) fail to lock the
config file. By configuring a timeout larger than zero, Git can be
told to try to lock the config again a couple times within the
specified timeout. If the timeout is configure to zero (which is the
default), Git will fail immediately when the config is already
locked.
40 changes: 39 additions & 1 deletion Documentation/scalar.txt
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,17 @@ scalar - A tool for managing large Git repositories
SYNOPSIS
--------
[verse]
scalar clone [--single-branch] [--branch <main-branch>] [--full-clone] <url> [<enlistment>]
scalar clone [--single-branch] [--branch <main-branch>] [--full-clone]
[--local-cache-path <path>] [--cache-server-url <url>] [--[no-]src]
<url> [<enlistment>]
scalar list
scalar register [<enlistment>]
scalar unregister [<enlistment>]
scalar run ( all | config | commit-graph | fetch | loose-objects | pack-files ) [<enlistment>]
scalar reconfigure [ --all | <enlistment> ]
scalar diagnose [<enlistment>]
scalar delete <enlistment>
scalar cache-server ( --get | --set <url> | --list [<remote>] ) [<enlistment>]

DESCRIPTION
-----------
Expand Down Expand Up @@ -80,10 +83,24 @@ remote-tracking branch for the branch this option was used for the initial
cloning. If the HEAD at the remote did not point at any branch when
`--single-branch` clone was made, no remote-tracking branch is created.

--no-src::
Skip adding a `src` directory within the target enlistment.

--[no-]full-clone::
A sparse-checkout is initialized by default. This behavior can be
turned off via `--full-clone`.

--local-cache-path <path>::
Override the path to the local cache root directory; Pre-fetched objects
are stored into a repository-dependent subdirectory of that path.
+
The default is `<drive>:\.scalarCache` on Windows (on the same drive as the
clone), and `~/.scalarCache` on macOS.

--cache-server-url <url>::
Retrieve missing objects from the specified remote, which is expected to
understand the GVFS protocol.

List
~~~~

Expand Down Expand Up @@ -157,6 +174,27 @@ delete <enlistment>::
This subcommand lets you delete an existing Scalar enlistment from your
local file system, unregistering the repository.

Cache-server
~~~~~~~~~~~~

cache-server ( --get | --set <url> | --list [<remote>] ) [<enlistment>]::
This command lets you query or set the GVFS-enabled cache server used
to fetch missing objects.

--get::
This is the default command mode: query the currently-configured cache
server URL, if any.

--list::
Access the `gvfs/info` endpoint of the specified remote (default:
`origin`) to figure out which cache servers are available, if any.
+
In contrast to the `--get` command mode (which only accesses the local
repository), this command mode triggers a request via the network that
potentially requires authentication. If authentication is required, the
configured credential helper is employed (see linkgit:git-credential[1]
for details).

SEE ALSO
--------
linkgit:git-clone[1], linkgit:git-maintenance[1].
Expand Down
3 changes: 2 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2737,6 +2737,7 @@ GIT_OBJS += git.o
.PHONY: git-objs
git-objs: $(GIT_OBJS)

SCALAR_OBJS := json-parser.o
SCALAR_OBJS += scalar.o
.PHONY: scalar-objs
scalar-objs: $(SCALAR_OBJS)
Expand Down Expand Up @@ -2885,7 +2886,7 @@ $(REMOTE_CURL_PRIMARY): remote-curl.o http.o http-walker.o $(LAZYLOAD_LIBCURL_OB
$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \
$(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS)

scalar$X: scalar.o GIT-LDFLAGS $(GITLIBS)
scalar$X: $(SCALAR_OBJS) GIT-LDFLAGS $(GITLIBS)
$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) \
$(filter %.o,$^) $(LIBS)

Expand Down
6 changes: 3 additions & 3 deletions abspath.c
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ int is_directory(const char *path)
}

/* removes the last path component from 'path' except if 'path' is root */
static void strip_last_component(struct strbuf *path)
void strip_last_path_component(struct strbuf *path)
{
size_t offset = offset_1st_component(path->buf);
size_t len = path->len;
Expand Down Expand Up @@ -119,7 +119,7 @@ static char *strbuf_realpath_1(struct strbuf *resolved, const char *path,
continue; /* '.' component */
} else if (next.len == 2 && !strcmp(next.buf, "..")) {
/* '..' component; strip the last path component */
strip_last_component(resolved);
strip_last_path_component(resolved);
continue;
}

Expand Down Expand Up @@ -171,7 +171,7 @@ static char *strbuf_realpath_1(struct strbuf *resolved, const char *path,
* strip off the last component since it will
* be replaced with the contents of the symlink
*/
strip_last_component(resolved);
strip_last_path_component(resolved);
}

/*
Expand Down
5 changes: 5 additions & 0 deletions abspath.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@ char *real_pathdup(const char *path, int die_on_error);
const char *absolute_path(const char *path);
char *absolute_pathdup(const char *path);

/**
* Remove the last path component from 'path' except if 'path' is root.
*/
void strip_last_path_component(struct strbuf *path);

/*
* Concatenate "prefix" (if len is non-zero) and "path", with no
* connecting characters (so "prefix" should end with a "/").
Expand Down
8 changes: 7 additions & 1 deletion config.c
Original file line number Diff line number Diff line change
Expand Up @@ -3415,6 +3415,7 @@ int git_config_set_multivar_in_file_gently(const char *config_filename,
const char *value_pattern,
unsigned flags)
{
static unsigned long timeout_ms = ULONG_MAX;
int fd = -1, in_fd = -1;
int ret;
struct lock_file lock = LOCK_INIT;
Expand All @@ -3433,11 +3434,16 @@ int git_config_set_multivar_in_file_gently(const char *config_filename,
if (!config_filename)
config_filename = filename_buf = git_pathdup("config");

if ((long)timeout_ms < 0 &&
git_config_get_ulong("core.configWriteLockTimeoutMS", &timeout_ms))
timeout_ms = 0;

/*
* The lock serves a purpose in addition to locking: the new
* contents of .git/config will be written into it.
*/
fd = hold_lock_file_for_update(&lock, config_filename, 0);
fd = hold_lock_file_for_update_timeout(&lock, config_filename, 0,
timeout_ms);
if (fd < 0) {
error_errno(_("could not lock config file %s"), config_filename);
ret = CONFIG_NO_LOCK;
Expand Down
2 changes: 1 addition & 1 deletion contrib/buildsystems/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -798,7 +798,7 @@ target_link_libraries(git-sh-i18n--envsubst common-main)
add_executable(git-shell ${CMAKE_SOURCE_DIR}/shell.c)
target_link_libraries(git-shell common-main)

add_executable(scalar ${CMAKE_SOURCE_DIR}/scalar.c)
add_executable(scalar ${CMAKE_SOURCE_DIR}/scalar.c ${CMAKE_SOURCE_DIR}/json-parser.c)
target_link_libraries(scalar common-main)

if(CURL_FOUND)
Expand Down
Loading

0 comments on commit 76b3495

Please sign in to comment.