Skip to content

Commit

Permalink
add project lock file to discourage concurrent use
Browse files Browse the repository at this point in the history
fixes #22 (sufficiently, for now)

* remove io/ioutil, comment on possible race conditions, update deps

* show DVC config in benchmarks, hide dep update from changelog
  • Loading branch information
kevin-hanselman committed Sep 24, 2021
1 parent 2bdf3b5 commit 968d5db
Show file tree
Hide file tree
Showing 48 changed files with 344 additions and 215 deletions.
1 change: 1 addition & 0 deletions .goreleaser.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,4 @@ changelog:
- '^docs:'
- '^test:'
- '^misc:'
- '^deps:'
16 changes: 8 additions & 8 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,22 +8,22 @@ require (
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2
github.com/cheggaaa/pb/v3 v3.0.8
github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
github.com/fatih/color v1.12.0 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/felixge/fgprof v0.9.1
github.com/google/go-cmp v0.5.6
github.com/google/pprof v0.0.0-20210726183535-c50bf4fe5303 // indirect
github.com/mattn/go-isatty v0.0.13
github.com/google/pprof v0.0.0-20210827144239-02619b876842 // indirect
github.com/mattn/go-isatty v0.0.14
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/mitchellh/go-homedir v1.1.0
github.com/pkg/errors v0.9.1
github.com/spf13/cast v1.4.0 // indirect
github.com/spf13/cobra v1.2.1
github.com/spf13/viper v1.8.1
github.com/spf13/viper v1.9.0
github.com/stretchr/objx v0.3.0 // indirect
github.com/stretchr/testify v1.7.0
github.com/zeebo/blake3 v0.2.0
github.com/zeebo/blake3 v0.2.1
go.uber.org/goleak v1.1.11
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
golang.org/x/text v0.3.6 // indirect
golang.org/x/sys v0.0.0-20210921065528-437939a70204 // indirect
golang.org/x/text v0.3.7 // indirect
gopkg.in/yaml.v2 v2.4.0
)
150 changes: 124 additions & 26 deletions go.sum

Large diffs are not rendered by default.

6 changes: 6 additions & 0 deletions integration/benchmarks/generate_front_matter.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,10 @@ cat << EOF
Dud version: $(dud version | xargs)
$(dvc version | head -n1 | xargs)
DVC non-default configuration:
EOF

# Add spaces to each line to treat the output as a Markdown code block.
dvc config --list --global | awk '{ print " " $0 }'
2 changes: 1 addition & 1 deletion integration/tests/basic_commit_copy/expected_fs.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[drwxr-xr-x user 4096] ./.dud/cache/49
[-r--r--r-- user 4] ./.dud/cache/49/dc870df1de7fd60794cebce449f5ccdae575affaa67a24b62acb03e039db92
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[drwxr-xr-x user 4096] ./.dud/cache/00
[-r--r--r-- user 2] ./.dud/cache/00/51fb8f5c8288b80163ea72ab2f482fc402ca9944b580aa57e694eedfc3ad1c
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[drwxr-xr-x user 4096] ./.dud/cache/00
[-r--r--r-- user 2] ./.dud/cache/00/51fb8f5c8288b80163ea72ab2f482fc402ca9944b580aa57e694eedfc3ad1c
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[drwxr-xr-x user 4096] ./.dud/cache/00
[-r--r--r-- user 2] ./.dud/cache/00/51fb8f5c8288b80163ea72ab2f482fc402ca9944b580aa57e694eedfc3ad1c
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[drwxr-xr-x user 4096] ./.dud/cache/00
[-r--r--r-- user 2] ./.dud/cache/00/51fb8f5c8288b80163ea72ab2f482fc402ca9944b580aa57e694eedfc3ad1c
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[drwxr-xr-x user 4096] ./.dud/cache/00
[-r--r--r-- user 2] ./.dud/cache/00/51fb8f5c8288b80163ea72ab2f482fc402ca9944b580aa57e694eedfc3ad1c
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[drwxr-xr-x user 4096] ./.dud/cache/00
[-r--r--r-- user 2] ./.dud/cache/00/51fb8f5c8288b80163ea72ab2f482fc402ca9944b580aa57e694eedfc3ad1c
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[drwxr-xr-x user 4096] ./.dud/cache/00
[-r--r--r-- user 2] ./.dud/cache/00/51fb8f5c8288b80163ea72ab2f482fc402ca9944b580aa57e694eedfc3ad1c
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[drwxr-xr-x user 4096] ./.dud/cache/00
[-r--r--r-- user 2] ./.dud/cache/00/51fb8f5c8288b80163ea72ab2f482fc402ca9944b580aa57e694eedfc3ad1c
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[drwxr-xr-x user 4096] ./.dud/cache/49
[-r--r--r-- user 4] ./.dud/cache/49/dc870df1de7fd60794cebce449f5ccdae575affaa67a24b62acb03e039db92
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[drwxr-xr-x user 4096] ./.dud/cache/49
[-r--r--r-- user 4] ./.dud/cache/49/dc870df1de7fd60794cebce449f5ccdae575affaa67a24b62acb03e039db92
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[drwxr-xr-x user 4096] ./.dud/cache/49
[-r--r--r-- user 4] ./.dud/cache/49/dc870df1de7fd60794cebce449f5ccdae575affaa67a24b62acb03e039db92
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[drwxr-xr-x user 4096] ./.dud/cache/49
[-r--r--r-- user 4] ./.dud/cache/49/dc870df1de7fd60794cebce449f5ccdae575affaa67a24b62acb03e039db92
Expand Down
2 changes: 1 addition & 1 deletion integration/tests/basic_run/00_base_stage/expected_fs.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[-rw-r--r-- user 621] ./.dud/config.yaml
[-rw-r--r-- user 10] ./.dud/index
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[-rw-r--r-- user 621] ./.dud/config.yaml
[-rw-r--r-- user 22] ./.dud/index
Expand Down
2 changes: 1 addition & 1 deletion integration/tests/external_cache/00_commit/expected_fs.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[-rw-r--r-- user 50] ./.dud/config.yaml
[-rw-r--r-- user 11] ./.dud/index
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[-rw-r--r-- user 50] ./.dud/config.yaml
[-rw-r--r-- user 11] ./.dud/index
Expand Down
30 changes: 30 additions & 0 deletions integration/tests/lock_project/run.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/bin/bash
set -euo pipefail

dud init

dud stage gen -o foo.txt -- sleep 10 > sleep.yaml

dud stage add sleep.yaml

# This loop is mainly to ensure this concurrent test code is working well.
for _ in $(seq 1 10); do
dud run &

# Give 'dud run' enough time to start
sleep 0.02

if dud status; then
echo 1>&2 'expected second concurrent dud command to fail'
exit 1
fi

# Kill dud run's command and wait for it stop. This should cause 'run' to
# fail, but we still expect Dud to release the project lock file.
killall sleep
wait
if test -f .dud/lock; then
echo 1>&2 'expected first dud command to clean up its lock file'
exit 1
fi
done
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[drwxr-xr-x user 4096] ./.dud/cache/00
[-rw-r--r-- user 2] ./.dud/cache/00/51fb8f5c8288b80163ea72ab2f482fc402ca9944b580aa57e694eedfc3ad1c
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[drwxr-xr-x user 4096] ./.dud/cache/00
[-rw-r--r-- user 2] ./.dud/cache/00/51fb8f5c8288b80163ea72ab2f482fc402ca9944b580aa57e694eedfc3ad1c
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[drwxr-xr-x user 4096] ./.dud/cache/00
[-rw-r--r-- user 2] ./.dud/cache/00/51fb8f5c8288b80163ea72ab2f482fc402ca9944b580aa57e694eedfc3ad1c
Expand Down
2 changes: 1 addition & 1 deletion integration/tests/pull/00_pull/expected_fs.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[drwxr-xr-x user 4096] ./.dud/cache/b3
[-r--r--r-- user 4] ./.dud/cache/b3/199d36d434044e6778b77d13f8dbaba32a73d9522c1ae8d0f73ef1ff14e71f
Expand Down
2 changes: 1 addition & 1 deletion integration/tests/pull/01_pull_copy/expected_fs.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[drwxr-xr-x user 4096] ./.dud/cache/b3
[-r--r--r-- user 4] ./.dud/cache/b3/199d36d434044e6778b77d13f8dbaba32a73d9522c1ae8d0f73ef1ff14e71f
Expand Down
2 changes: 1 addition & 1 deletion integration/tests/pull/02_pull_stage/expected_fs.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[drwxr-xr-x user 4096] ./.dud/cache/b3
[-r--r--r-- user 4] ./.dud/cache/b3/199d36d434044e6778b77d13f8dbaba32a73d9522c1ae8d0f73ef1ff14e71f
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.
[drwxr-xr-x user 4096] ./.dud
[-rw-r--r-- user 7] ./.dud/.gitignore
[-rw-r--r-- user 14] ./.dud/.gitignore
[drwxr-xr-x user 4096] ./.dud/cache
[drwxr-xr-x user 4096] ./.dud/cache/53
[-r--r--r-- user 7] ./.dud/cache/53/4659321d2eea6b13aea4f4c94c3b4f624622295da31506722b47a8eb9d726c
Expand Down
14 changes: 10 additions & 4 deletions src/cache/commit.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"context"
"encoding/json"
"io"
"io/ioutil"
"os"
"path/filepath"

Expand Down Expand Up @@ -63,12 +62,12 @@ func (ch LocalCache) Commit(

var canRenameFileBetweenDirs = func(srcDir, dstDir string) (bool, error) {
// Touch a file in each directory.
srcFile, err := ioutil.TempFile(srcDir, "")
srcFile, err := os.CreateTemp(srcDir, "")
if err != nil {
return false, err
}
srcFile.Close()
dstFile, err := ioutil.TempFile(dstDir, "")
dstFile, err := os.CreateTemp(dstDir, "")
if err != nil {
return false, err
}
Expand Down Expand Up @@ -166,7 +165,7 @@ func (ch LocalCache) commitBytes(reader io.Reader, moveFile string) (string, err
// If there's no file we can move, we need to copy the bytes from reader to
// the cache.
if moveFile == "" {
tempFile, err := ioutil.TempFile(ch.dir, "")
tempFile, err := os.CreateTemp(ch.dir, "")
if err != nil {
return "", err
}
Expand All @@ -188,6 +187,13 @@ func (ch LocalCache) commitBytes(reader io.Reader, moveFile string) (string, err
if err = os.MkdirAll(dstDir, 0o755); err != nil {
return "", err
}
// This rename may race others, but luckily we don't care who wins the
// race. Everyone in the race is trying to put the same exact file in the
// cache (because of content-addressed storage), so the outcome is the same
// no matter who wins the race. At the OS level rename is atomic, so
// there's no risk of corrupting the destination file with multiple
// concurrent syscalls. (This is at least true for UNIX, but that's all we
// support. See also: https://github.com/golang/go/issues/8914)
if err = os.Rename(moveFile, cachePath); err != nil {
return "", err
}
Expand Down
3 changes: 3 additions & 0 deletions src/cache/dir_checkout_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,16 @@ import (
"github.com/kevin-hanselman/dud/src/artifact"
"github.com/kevin-hanselman/dud/src/fsutil"
"github.com/kevin-hanselman/dud/src/strategy"
"go.uber.org/goleak"
)

func TestDirectoryCheckoutIntegration(t *testing.T) {
if testing.Short() {
t.Skip()
}

defer goleak.VerifyNone(t)

logger := agglog.NewNullLogger()

maxSharedWorkers = 1
Expand Down
8 changes: 5 additions & 3 deletions src/cache/dir_commit_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package cache

import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"testing"
Expand All @@ -13,13 +12,16 @@ import (
"github.com/kevin-hanselman/dud/src/fsutil"
"github.com/kevin-hanselman/dud/src/strategy"
"github.com/kevin-hanselman/dud/src/testutil"
"go.uber.org/goleak"
)

func TestDirectoryCommitIntegration(t *testing.T) {
if testing.Short() {
t.Skip()
}

defer goleak.VerifyNone(t)

makeExpectedStatus := func(art artifact.Artifact) artifact.Status {
upToDate := func(art artifact.Artifact) *artifact.Status {
return &artifact.Status{
Expand Down Expand Up @@ -187,7 +189,7 @@ func setupDirTest(t *testing.T) (testutil.TempDirs, artifact.Artifact, LocalCach
for i := 1; i <= 5; i++ {
s := fmt.Sprint(i)
path := filepath.Join(dirs.WorkDir, "foo", fmt.Sprintf("%s.txt", s))
if err := ioutil.WriteFile(path, []byte(s), 0o644); err != nil {
if err := os.WriteFile(path, []byte(s), 0o644); err != nil {
t.Fatal(err)
}
}
Expand All @@ -196,7 +198,7 @@ func setupDirTest(t *testing.T) (testutil.TempDirs, artifact.Artifact, LocalCach
for i := 4; i <= 8; i++ {
s := fmt.Sprint(i)
path := filepath.Join(dirs.WorkDir, "foo", "bar", fmt.Sprintf("%s.txt", s))
if err := ioutil.WriteFile(path, []byte(s), 0o644); err != nil {
if err := os.WriteFile(path, []byte(s), 0o644); err != nil {
t.Fatal(err)
}
}
Expand Down
2 changes: 2 additions & 0 deletions src/cache/dir_status_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,14 @@ import (
"github.com/kevin-hanselman/dud/src/fsutil"

"github.com/google/go-cmp/cmp"
"go.uber.org/goleak"
)

func TestDirectoryStatusIntegration(t *testing.T) {
if testing.Short() {
t.Skip()
}
defer goleak.VerifyNone(t)

notCommitted := func(art artifact.Artifact) *artifact.Status {
return &artifact.Status{
Expand Down
Loading

0 comments on commit 968d5db

Please sign in to comment.