Skip to content

Commit

Permalink
Merge pull request #371 from dscho/run-scalar-functional-tests-and-fi…
Browse files Browse the repository at this point in the history
…x-built-in-fsmonitor

Fix the built-in FSMonitor, and run Scalar's Functional Tests as part of the automated builds
  • Loading branch information
dscho committed Oct 8, 2024
2 parents 6f1759d + e34dfb6 commit 2d9c105
Show file tree
Hide file tree
Showing 21 changed files with 1,758 additions and 44 deletions.
220 changes: 220 additions & 0 deletions .github/workflows/scalar-functional-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,220 @@
name: Scalar Functional Tests

env:
SCALAR_REPOSITORY: microsoft/scalar
SCALAR_REF: main
DEBUG_WITH_TMATE: false
SCALAR_TEST_SKIP_VSTS_INFO: true

on:
push:
branches: [ vfs-*, tentative/vfs-* ]
pull_request:
branches: [ vfs-*, features/* ]

jobs:
scalar:
name: "Scalar Functional Tests"

strategy:
fail-fast: false
matrix:
# Order by runtime (in descending order)
os: [windows-2019, macos-13, ubuntu-20.04, ubuntu-22.04]
# Scalar.NET used to be tested using `features: [false, experimental]`
# But currently, Scalar/C ignores `feature.scalar` altogether, so let's
# save some electrons and run only one of them...
features: [ignored]
exclude:
# The built-in FSMonitor is not (yet) supported on Linux
- os: ubuntu-20.04
features: experimental
- os: ubuntu-22.04
features: experimental
runs-on: ${{ matrix.os }}

env:
BUILD_FRAGMENT: bin/Release/netcoreapp3.1
GIT_FORCE_UNTRACKED_CACHE: 1

steps:
- name: Check out Git's source code
uses: actions/checkout@v3

- name: Setup build tools on Windows
if: runner.os == 'Windows'
uses: git-for-windows/setup-git-for-windows-sdk@v1

- name: Provide a minimal `install` on Windows
if: runner.os == 'Windows'
shell: bash
run: |
test -x /usr/bin/install ||
tr % '\t' >/usr/bin/install <<-\EOF
#!/bin/sh
cmd=cp
while test $# != 0
do
%case "$1" in
%-d) cmd="mkdir -p";;
%-m) shift;; # ignore mode
%*) break;;
%esac
%shift
done
exec $cmd "$@"
EOF
- name: Install build dependencies for Git (Linux)
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get -q -y install libssl-dev libcurl4-openssl-dev gettext
- name: Build and install Git
shell: bash
env:
NO_TCLTK: Yup
run: |
# We do require a VFS version
def_ver="$(sed -n 's/DEF_VER=\(.*vfs.*\)/\1/p' GIT-VERSION-GEN)"
test -n "$def_ver"
# Ensure that `git version` reflects DEF_VER
case "$(git describe --match "v[0-9]*vfs*" HEAD)" in
${def_ver%%.vfs.*}.vfs.*) ;; # okay, we can use this
*) git -c user.name=ci -c user.email=ci@github tag -m for-testing ${def_ver}.NNN.g$(git rev-parse --short HEAD);;
esac
SUDO=
extra=
case "${{ runner.os }}" in
Windows)
extra=DESTDIR=/c/Progra~1/Git
cygpath -aw "/c/Program Files/Git/cmd" >>$GITHUB_PATH
;;
Linux)
SUDO=sudo
extra=prefix=/usr
;;
macOS)
SUDO=sudo
extra=prefix=/usr/local
;;
esac
$SUDO make -j5 $extra install
- name: Ensure that we use the built Git and Scalar
shell: bash
run: |
type -p git
git version
case "$(git version)" in *.vfs.*) echo Good;; *) exit 1;; esac
type -p scalar
scalar version
case "$(scalar version 2>&1)" in *.vfs.*) echo Good;; *) exit 1;; esac
- name: Check out Scalar's source code
uses: actions/checkout@v3
with:
fetch-depth: 0 # Indicate full history so Nerdbank.GitVersioning works.
path: scalar
repository: ${{ env.SCALAR_REPOSITORY }}
ref: ${{ env.SCALAR_REF }}

- name: Setup .NET Core
uses: actions/setup-dotnet@v3
with:
dotnet-version: '3.1.x'

- name: Install dependencies
run: dotnet restore
working-directory: scalar
env:
DOTNET_NOLOGO: 1

- name: Build
working-directory: scalar
run: dotnet build --configuration Release --no-restore -p:UseAppHost=true # Force generation of executable on macOS.

- name: Setup platform (Linux)
if: runner.os == 'Linux'
run: |
echo "BUILD_PLATFORM=${{ runner.os }}" >>$GITHUB_ENV
echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$GITHUB_ENV
- name: Setup platform (Mac)
if: runner.os == 'macOS'
run: |
echo 'BUILD_PLATFORM=Mac' >>$GITHUB_ENV
echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$GITHUB_ENV
- name: Setup platform (Windows)
if: runner.os == 'Windows'
run: |
echo "BUILD_PLATFORM=${{ runner.os }}" >>$env:GITHUB_ENV
echo 'BUILD_FILE_EXT=.exe' >>$env:GITHUB_ENV
echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$env:GITHUB_ENV
- name: Configure feature.scalar
run: git config --global feature.scalar ${{ matrix.features }}

- id: functional_test
name: Functional test
timeout-minutes: 60
working-directory: scalar
shell: bash
run: |
export GIT_TRACE2_EVENT="$PWD/$TRACE2_BASENAME/Event"
export GIT_TRACE2_PERF="$PWD/$TRACE2_BASENAME/Perf"
export GIT_TRACE2_EVENT_BRIEF=true
export GIT_TRACE2_PERF_BRIEF=true
mkdir -p "$TRACE2_BASENAME"
mkdir -p "$TRACE2_BASENAME/Event"
mkdir -p "$TRACE2_BASENAME/Perf"
git version --build-options
cd ../out
Scalar.FunctionalTests/$BUILD_FRAGMENT/Scalar.FunctionalTests$BUILD_FILE_EXT --test-scalar-on-path --test-git-on-path --timeout=300000 --full-suite
- name: Force-stop FSMonitor daemons and Git processes (Windows)
if: runner.os == 'Windows' && (success() || failure())
shell: bash
run: |
set -x
wmic process get CommandLine,ExecutablePath,HandleCount,Name,ParentProcessID,ProcessID
wmic process where "CommandLine Like '%fsmonitor--daemon %run'" delete
wmic process where "ExecutablePath Like '%git.exe'" delete
- id: trace2_zip_unix
if: runner.os != 'Windows' && ( success() || failure() ) && ( steps.functional_test.conclusion == 'success' || steps.functional_test.conclusion == 'failure' )
name: Zip Trace2 Logs (Unix)
shell: bash
working-directory: scalar
run: zip -q -r $TRACE2_BASENAME.zip $TRACE2_BASENAME/

- id: trace2_zip_windows
if: runner.os == 'Windows' && ( success() || failure() ) && ( steps.functional_test.conclusion == 'success' || steps.functional_test.conclusion == 'failure' )
name: Zip Trace2 Logs (Windows)
working-directory: scalar
run: Compress-Archive -DestinationPath ${{ env.TRACE2_BASENAME }}.zip -Path ${{ env.TRACE2_BASENAME }}

- name: Archive Trace2 Logs
if: ( success() || failure() ) && ( steps.trace2_zip_unix.conclusion == 'success' || steps.trace2_zip_windows.conclusion == 'success' )
uses: actions/upload-artifact@v3
with:
name: ${{ env.TRACE2_BASENAME }}.zip
path: scalar/${{ env.TRACE2_BASENAME }}.zip
retention-days: 3

# The GitHub Action `action-tmate` allows developers to connect to the running agent
# using SSH (it will be a `tmux` session; on Windows agents it will be inside the MSYS2
# environment in `C:\msys64`, therefore it can be slightly tricky to interact with
# Git for Windows, which runs a slightly incompatible MSYS2 runtime).
- name: action-tmate
if: env.DEBUG_WITH_TMATE == 'true' && failure()
uses: mxschmitt/action-tmate@v3
with:
limit-access-to-actor: true
9 changes: 9 additions & 0 deletions Documentation/config/core.txt
Original file line number Diff line number Diff line change
Expand Up @@ -834,3 +834,12 @@ core.WSLCompat::
The default value is false. When set to true, Git will set the mode
bits of the file in the way of wsl, so that the executable flag of
files can be set or read correctly.

core.configWriteLockTimeoutMS::
When processes try to write to the config concurrently, it is likely
that one process "wins" and the other process(es) fail to lock the
config file. By configuring a timeout larger than zero, Git can be
told to try to lock the config again a couple times within the
specified timeout. If the timeout is configure to zero (which is the
default), Git will fail immediately when the config is already
locked.
56 changes: 55 additions & 1 deletion Documentation/scalar.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,16 @@ SYNOPSIS
--------
[verse]
scalar clone [--single-branch] [--branch <main-branch>] [--full-clone]
[--[no-]src] <url> [<enlistment>]
[--[no-]src] [--local-cache-path <path>] [--cache-server-url <url>]
<url> [<enlistment>]
scalar list
scalar register [<enlistment>]
scalar unregister [<enlistment>]
scalar run ( all | config | commit-graph | fetch | loose-objects | pack-files ) [<enlistment>]
scalar reconfigure [ --all | <enlistment> ]
scalar diagnose [<enlistment>]
scalar delete <enlistment>
scalar cache-server ( --get | --set <url> | --list [<remote>] ) [<enlistment>]

DESCRIPTION
-----------
Expand Down Expand Up @@ -97,6 +99,37 @@ cloning. If the HEAD at the remote did not point at any branch when
A sparse-checkout is initialized by default. This behavior can be
turned off via `--full-clone`.

--local-cache-path <path>::
Override the path to the local cache root directory; Pre-fetched objects
are stored into a repository-dependent subdirectory of that path.
+
The default is `<drive>:\.scalarCache` on Windows (on the same drive as the
clone), and `~/.scalarCache` on macOS.

--cache-server-url <url>::
Retrieve missing objects from the specified remote, which is expected to
understand the GVFS protocol.

--[no-]gvfs-protocol::
When cloning from a `<url>` with either `dev.azure.com` or
`visualstudio.com` in the name, `scalar clone` will attempt to use the GVFS
Protocol to access Git objects, specifically from a cache server when
available, and will fail to clone if there is an error over that protocol.

To enable the GVFS Protocol regardless of the origin `<url>`, use
`--gvfs-protocol`. This will cause `scalar clone` to fail when the origin
server fails to provide a valid response to the `gvfs/config` endpoint.

To disable the GVFS Protocol, use `--no-gvfs-protocol` and `scalar clone`
will only use the Git protocol, starting with a partial clone. This can be
helpful if your `<url>` points to Azure Repos but the repository does not
have GVFS cache servers enabled. It is likely more efficient to use its
partial clone functionality through the Git protocol.

Previous versions of `scalar clone` could fall back to a partial clone over
the Git protocol if there is any issue gathering GVFS configuration
information from the origin server.

List
~~~~

Expand Down Expand Up @@ -170,6 +203,27 @@ delete <enlistment>::
This subcommand lets you delete an existing Scalar enlistment from your
local file system, unregistering the repository.

Cache-server
~~~~~~~~~~~~

cache-server ( --get | --set <url> | --list [<remote>] ) [<enlistment>]::
This command lets you query or set the GVFS-enabled cache server used
to fetch missing objects.

--get::
This is the default command mode: query the currently-configured cache
server URL, if any.

--list::
Access the `gvfs/info` endpoint of the specified remote (default:
`origin`) to figure out which cache servers are available, if any.
+
In contrast to the `--get` command mode (which only accesses the local
repository), this command mode triggers a request via the network that
potentially requires authentication. If authentication is required, the
configured credential helper is employed (see linkgit:git-credential[1]
for details).

SEE ALSO
--------
linkgit:git-clone[1], linkgit:git-maintenance[1].
Expand Down
3 changes: 2 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2820,6 +2820,7 @@ GIT_OBJS += git.o
.PHONY: git-objs
git-objs: $(GIT_OBJS)

SCALAR_OBJS := json-parser.o
SCALAR_OBJS += scalar.o
.PHONY: scalar-objs
scalar-objs: $(SCALAR_OBJS)
Expand Down Expand Up @@ -2971,7 +2972,7 @@ $(REMOTE_CURL_PRIMARY): remote-curl.o http.o http-walker.o $(LAZYLOAD_LIBCURL_OB
$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \
$(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS)

scalar$X: scalar.o GIT-LDFLAGS $(GITLIBS)
scalar$X: $(SCALAR_OBJS) GIT-LDFLAGS $(GITLIBS)
$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) \
$(filter %.o,$^) $(LIBS)

Expand Down
6 changes: 3 additions & 3 deletions abspath.c
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ int is_directory(const char *path)
}

/* removes the last path component from 'path' except if 'path' is root */
static void strip_last_component(struct strbuf *path)
void strip_last_path_component(struct strbuf *path)
{
size_t offset = offset_1st_component(path->buf);
size_t len = path->len;
Expand Down Expand Up @@ -119,7 +119,7 @@ static char *strbuf_realpath_1(struct strbuf *resolved, const char *path,
continue; /* '.' component */
} else if (next.len == 2 && !strcmp(next.buf, "..")) {
/* '..' component; strip the last path component */
strip_last_component(resolved);
strip_last_path_component(resolved);
continue;
}

Expand Down Expand Up @@ -171,7 +171,7 @@ static char *strbuf_realpath_1(struct strbuf *resolved, const char *path,
* strip off the last component since it will
* be replaced with the contents of the symlink
*/
strip_last_component(resolved);
strip_last_path_component(resolved);
}

/*
Expand Down
5 changes: 5 additions & 0 deletions abspath.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@ char *real_pathdup(const char *path, int die_on_error);
const char *absolute_path(const char *path);
char *absolute_pathdup(const char *path);

/**
* Remove the last path component from 'path' except if 'path' is root.
*/
void strip_last_path_component(struct strbuf *path);

/*
* Concatenate "prefix" (if len is non-zero) and "path", with no
* connecting characters (so "prefix" should end with a "/").
Expand Down
8 changes: 7 additions & 1 deletion config.c
Original file line number Diff line number Diff line change
Expand Up @@ -3296,6 +3296,7 @@ int repo_config_set_multivar_in_file_gently(struct repository *r,
const char *comment,
unsigned flags)
{
static unsigned long timeout_ms = ULONG_MAX;
int fd = -1, in_fd = -1;
int ret;
struct lock_file lock = LOCK_INIT;
Expand All @@ -3316,11 +3317,16 @@ int repo_config_set_multivar_in_file_gently(struct repository *r,
if (!config_filename)
config_filename = filename_buf = repo_git_path(r, "config");

if ((long)timeout_ms < 0 &&
git_config_get_ulong("core.configWriteLockTimeoutMS", &timeout_ms))
timeout_ms = 0;

/*
* The lock serves a purpose in addition to locking: the new
* contents of .git/config will be written into it.
*/
fd = hold_lock_file_for_update(&lock, config_filename, 0);
fd = hold_lock_file_for_update_timeout(&lock, config_filename, 0,
timeout_ms);
if (fd < 0) {
error_errno(_("could not lock config file %s"), config_filename);
ret = CONFIG_NO_LOCK;
Expand Down
Loading

0 comments on commit 2d9c105

Please sign in to comment.