Skip to content

Commit

Permalink
Merge pull request #371 from dscho/run-scalar-functional-tests-and-fi…
Browse files Browse the repository at this point in the history
…x-built-in-fsmonitor

Fix the built-in FSMonitor, and run Scalar's Functional Tests as part of the automated builds
  • Loading branch information
dscho committed Sep 22, 2022
2 parents 1e9f127 + 11e4364 commit ee8907d
Show file tree
Hide file tree
Showing 27 changed files with 1,771 additions and 83 deletions.
15 changes: 15 additions & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,13 @@ jobs:
HOME: ${{runner.workspace}}
NO_PERL: 1
run: . /etc/profile && ci/make-test-artifacts.sh artifacts
- name: build Scalar
shell: bash
run: |
make -C contrib/scalar &&
mkdir -p artifacts/bin-wrappers artifacts/contrib/scalar &&
cp contrib/scalar/scalar.exe artifacts/contrib/scalar/ &&
cp bin-wrappers/scalar artifacts/bin-wrappers/
- name: zip up tracked files
run: git archive -o artifacts/tracked.tar.gz HEAD
- name: upload tracked files and build artifacts
Expand Down Expand Up @@ -160,6 +167,8 @@ jobs:
run: compat\vcbuild\vcpkg_copy_dlls.bat release ${{ matrix.arch }}-windows
- name: generate Visual Studio solution
shell: bash
env:
INCLUDE_SCALAR: YesPlease
run: |
cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/${{ matrix.arch }}-windows \
-DNO_GETTEXT=YesPlease -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON -DCMAKE_GENERATOR_PLATFORM=${{ matrix.arch }} -DVCPKG_ARCH=${{ matrix.arch }}-windows -DHOST_CPU=${{ matrix.arch }}
Expand All @@ -173,6 +182,12 @@ jobs:
run: |
mkdir -p artifacts &&
eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts NO_GETTEXT=YesPlease 2>&1 | grep ^tar)"
- name: copy Scalar
shell: bash
run: |
mkdir -p artifacts/bin-wrappers artifacts/contrib/scalar &&
cp contrib/scalar/scalar.exe artifacts/contrib/scalar/ &&
cp bin-wrappers/scalar artifacts/bin-wrappers/
- name: zip up tracked files
run: git archive -o artifacts/tracked.tar.gz HEAD
- name: upload tracked files and build artifacts
Expand Down
220 changes: 220 additions & 0 deletions .github/workflows/scalar-functional-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,220 @@
name: Scalar Functional Tests

env:
SCALAR_REPOSITORY: microsoft/scalar
SCALAR_REF: main
DEBUG_WITH_TMATE: false
SCALAR_TEST_SKIP_VSTS_INFO: true

on:
push:
branches: [ vfs-*, tentative/vfs-* ]
pull_request:
branches: [ vfs-*, features/* ]

jobs:
scalar:
name: "Scalar Functional Tests"

strategy:
fail-fast: false
matrix:
# Order by runtime (in descending order)
os: [windows-2019, macos-10.15, ubuntu-18.04, ubuntu-20.04]
# Scalar.NET used to be tested using `features: [false, experimental]`
# But currently, Scalar/C ignores `feature.scalar` altogether, so let's
# save some electrons and run only one of them...
features: [ignored]
exclude:
# The built-in FSMonitor is not (yet) supported on Linux
- os: ubuntu-18.04
features: experimental
- os: ubuntu-20.04
features: experimental
runs-on: ${{ matrix.os }}

env:
BUILD_FRAGMENT: bin/Release/netcoreapp3.1
GIT_FORCE_UNTRACKED_CACHE: 1

steps:
- name: Check out Git's source code
uses: actions/checkout@v2

- name: Setup build tools on Windows
if: runner.os == 'Windows'
uses: git-for-windows/setup-git-for-windows-sdk@v1

- name: Provide a minimal `install` on Windows
if: runner.os == 'Windows'
shell: bash
run: |
test -x /usr/bin/install ||
tr % '\t' >/usr/bin/install <<-\EOF
#!/bin/sh
cmd=cp
while test $# != 0
do
%case "$1" in
%-d) cmd="mkdir -p";;
%-m) shift;; # ignore mode
%*) break;;
%esac
%shift
done
exec $cmd "$@"
EOF
- name: Install build dependencies for Git (Linux)
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get -q -y install libssl-dev libcurl4-openssl-dev gettext
- name: Build and install Git
shell: bash
env:
NO_TCLTK: Yup
run: |
# We do require a VFS version
def_ver="$(sed -n 's/DEF_VER=\(.*vfs.*\)/\1/p' GIT-VERSION-GEN)"
test -n "$def_ver"
# Ensure that `git version` reflects DEF_VER
case "$(git describe --match "v[0-9]*vfs*" HEAD)" in
${def_ver%%.vfs.*}.vfs.*) ;; # okay, we can use this
*) git -c user.name=ci -c user.email=ci@github tag -m for-testing ${def_ver}.NNN.g$(git rev-parse --short HEAD);;
esac
SUDO=
extra=
case "${{ runner.os }}" in
Windows)
extra=DESTDIR=/c/Progra~1/Git
cygpath -aw "/c/Program Files/Git/cmd" >>$GITHUB_PATH
;;
Linux)
SUDO=sudo
extra=prefix=/usr
;;
macOS)
SUDO=sudo
extra=prefix=/usr/local
;;
esac
$SUDO make -j5 $extra install
- name: Ensure that we use the built Git and Scalar
shell: bash
run: |
type -p git
git version
case "$(git version)" in *.vfs.*) echo Good;; *) exit 1;; esac
type -p scalar
scalar version
case "$(scalar version 2>&1)" in *.vfs.*) echo Good;; *) exit 1;; esac
- name: Check out Scalar's source code
uses: actions/checkout@v2
with:
fetch-depth: 0 # Indicate full history so Nerdbank.GitVersioning works.
path: scalar
repository: ${{ env.SCALAR_REPOSITORY }}
ref: ${{ env.SCALAR_REF }}

- name: Setup .NET Core
uses: actions/setup-dotnet@v1
with:
dotnet-version: 3.1.302

- name: Install dependencies
run: dotnet restore
working-directory: scalar
env:
DOTNET_NOLOGO: 1

- name: Build
working-directory: scalar
run: dotnet build --configuration Release --no-restore -p:UseAppHost=true # Force generation of executable on macOS.

- name: Setup platform (Linux)
if: runner.os == 'Linux'
run: |
echo "BUILD_PLATFORM=${{ runner.os }}" >>$GITHUB_ENV
echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$GITHUB_ENV
- name: Setup platform (Mac)
if: runner.os == 'macOS'
run: |
echo 'BUILD_PLATFORM=Mac' >>$GITHUB_ENV
echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$GITHUB_ENV
- name: Setup platform (Windows)
if: runner.os == 'Windows'
run: |
echo "BUILD_PLATFORM=${{ runner.os }}" >>$env:GITHUB_ENV
echo 'BUILD_FILE_EXT=.exe' >>$env:GITHUB_ENV
echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$env:GITHUB_ENV
- name: Configure feature.scalar
run: git config --global feature.scalar ${{ matrix.features }}

- id: functional_test
name: Functional test
timeout-minutes: 60
working-directory: scalar
shell: bash
run: |
export GIT_TRACE2_EVENT="$PWD/$TRACE2_BASENAME/Event"
export GIT_TRACE2_PERF="$PWD/$TRACE2_BASENAME/Perf"
export GIT_TRACE2_EVENT_BRIEF=true
export GIT_TRACE2_PERF_BRIEF=true
mkdir -p "$TRACE2_BASENAME"
mkdir -p "$TRACE2_BASENAME/Event"
mkdir -p "$TRACE2_BASENAME/Perf"
git version --build-options
cd ../out
Scalar.FunctionalTests/$BUILD_FRAGMENT/Scalar.FunctionalTests$BUILD_FILE_EXT --test-scalar-on-path --test-git-on-path --timeout=300000 --full-suite
- name: Force-stop FSMonitor daemons and Git processes (Windows)
if: runner.os == 'Windows' && (success() || failure())
shell: bash
run: |
set -x
wmic process get CommandLine,ExecutablePath,HandleCount,Name,ParentProcessID,ProcessID
wmic process where "CommandLine Like '%fsmonitor--daemon %run'" delete
wmic process where "ExecutablePath Like '%git.exe'" delete
- id: trace2_zip_unix
if: runner.os != 'Windows' && ( success() || failure() ) && ( steps.functional_test.conclusion == 'success' || steps.functional_test.conclusion == 'failure' )
name: Zip Trace2 Logs (Unix)
shell: bash
working-directory: scalar
run: zip -q -r $TRACE2_BASENAME.zip $TRACE2_BASENAME/

- id: trace2_zip_windows
if: runner.os == 'Windows' && ( success() || failure() ) && ( steps.functional_test.conclusion == 'success' || steps.functional_test.conclusion == 'failure' )
name: Zip Trace2 Logs (Windows)
working-directory: scalar
run: Compress-Archive -DestinationPath ${{ env.TRACE2_BASENAME }}.zip -Path ${{ env.TRACE2_BASENAME }}

- name: Archive Trace2 Logs
if: ( success() || failure() ) && ( steps.trace2_zip_unix.conclusion == 'success' || steps.trace2_zip_windows.conclusion == 'success' )
uses: actions/upload-artifact@v2
with:
name: ${{ env.TRACE2_BASENAME }}.zip
path: scalar/${{ env.TRACE2_BASENAME }}.zip
retention-days: 3

# The GitHub Action `action-tmate` allows developers to connect to the running agent
# using SSH (it will be a `tmux` session; on Windows agents it will be inside the MSYS2
# environment in `C:\msys64`, therefore it can be slightly tricky to interact with
# Git for Windows, which runs a slightly incompatible MSYS2 runtime).
- name: action-tmate
if: env.DEBUG_WITH_TMATE == 'true' && failure()
uses: mxschmitt/action-tmate@v3
with:
limit-access-to-actor: true
9 changes: 9 additions & 0 deletions Documentation/config/core.txt
Original file line number Diff line number Diff line change
Expand Up @@ -806,3 +806,12 @@ core.abbrev::
If set to "no", no abbreviation is made and the object names
are shown in their full length.
The minimum length is 4.

core.configWriteLockTimeoutMS::
When processes try to write to the config concurrently, it is likely
that one process "wins" and the other process(es) fail to lock the
config file. By configuring a timeout larger than zero, Git can be
told to try to lock the config again a couple times within the
specified timeout. If the timeout is configure to zero (which is the
default), Git will fail immediately when the config is already
locked.
37 changes: 36 additions & 1 deletion Documentation/scalar.txt
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,17 @@ scalar - A tool for managing large Git repositories
SYNOPSIS
--------
[verse]
scalar clone [--single-branch] [--branch <main-branch>] [--full-clone] <url> [<enlistment>]
scalar clone [--single-branch] [--branch <main-branch>] [--full-clone]
[--local-cache-path <path>] [--cache-server-url <url>]
<url> [<enlistment>]
scalar list
scalar register [<enlistment>]
scalar unregister [<enlistment>]
scalar run ( all | config | commit-graph | fetch | loose-objects | pack-files ) [<enlistment>]
scalar reconfigure [ --all | <enlistment> ]
scalar diagnose [<enlistment>]
scalar delete <enlistment>
scalar cache-server ( --get | --set <url> | --list [<remote>] ) [<enlistment>]

DESCRIPTION
-----------
Expand Down Expand Up @@ -84,6 +87,17 @@ cloning. If the HEAD at the remote did not point at any branch when
A sparse-checkout is initialized by default. This behavior can be
turned off via `--full-clone`.

--local-cache-path <path>::
Override the path to the local cache root directory; Pre-fetched objects
are stored into a repository-dependent subdirectory of that path.
+
The default is `<drive>:\.scalarCache` on Windows (on the same drive as the
clone), and `~/.scalarCache` on macOS.

--cache-server-url <url>::
Retrieve missing objects from the specified remote, which is expected to
understand the GVFS protocol.

List
~~~~

Expand Down Expand Up @@ -157,6 +171,27 @@ delete <enlistment>::
This subcommand lets you delete an existing Scalar enlistment from your
local file system, unregistering the repository.

Cache-server
~~~~~~~~~~~~

cache-server ( --get | --set <url> | --list [<remote>] ) [<enlistment>]::
This command lets you query or set the GVFS-enabled cache server used
to fetch missing objects.

--get::
This is the default command mode: query the currently-configured cache
server URL, if any.

--list::
Access the `gvfs/info` endpoint of the specified remote (default:
`origin`) to figure out which cache servers are available, if any.
+
In contrast to the `--get` command mode (which only accesses the local
repository), this command mode triggers a request via the network that
potentially requires authentication. If authentication is required, the
configured credential helper is employed (see linkgit:git-credential[1]
for details).

SEE ALSO
--------
linkgit:git-clone[1], linkgit:git-maintenance[1].
Expand Down
3 changes: 2 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2604,6 +2604,7 @@ GIT_OBJS += git.o
.PHONY: git-objs
git-objs: $(GIT_OBJS)

SCALAR_OBJS := json-parser.o
SCALAR_OBJS += scalar.o
.PHONY: scalar-objs
scalar-objs: $(SCALAR_OBJS)
Expand Down Expand Up @@ -2752,7 +2753,7 @@ $(REMOTE_CURL_PRIMARY): remote-curl.o http.o http-walker.o GIT-LDFLAGS $(GITLIBS
$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \
$(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS)

scalar$X: scalar.o GIT-LDFLAGS $(GITLIBS)
scalar$X: $(SCALAR_OBJS) GIT-LDFLAGS $(GITLIBS)
$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) \
$(filter %.o,$^) $(LIBS)

Expand Down
24 changes: 19 additions & 5 deletions builtin/gc.c
Original file line number Diff line number Diff line change
Expand Up @@ -1550,21 +1550,35 @@ static int maintenance_unregister(int argc, const char **argv, const char *prefi
struct option options[] = {
OPT_END(),
};
int rc;
const char *key = "maintenance.repo";
int rc = 0;
struct child_process config_unset = CHILD_PROCESS_INIT;
char *maintpath = get_maintpath();
int found = 0;
struct string_list_item *item;
const struct string_list *list = git_config_get_value_multi(key);

argc = parse_options(argc, argv, prefix, options,
builtin_maintenance_unregister_usage, 0);
if (argc)
usage_with_options(builtin_maintenance_unregister_usage,
options);

config_unset.git_cmd = 1;
strvec_pushl(&config_unset.args, "config", "--global", "--unset",
"--fixed-value", "maintenance.repo", maintpath, NULL);
for_each_string_list_item(item, list) {
if (!strcmp(maintpath, item->string)) {
found = 1;
break;
}
}

if (found) {
config_unset.git_cmd = 1;
strvec_pushl(&config_unset.args, "config", "--global", "--unset",
"--fixed-value", key, maintpath, NULL);

rc = run_command(&config_unset);
}

rc = run_command(&config_unset);
free(maintpath);
return rc;
}
Expand Down
Loading

0 comments on commit ee8907d

Please sign in to comment.