Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

respin glukasik's cgo zstd PR #559

Merged
merged 11 commits into from
Jul 17, 2022
Merged

respin glukasik's cgo zstd PR #559

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
19f7ebe
[zstd] add -zstd_implementation=cgo option
glukasiknuro Jun 28, 2022
94a4346
[zstd] make zstdimpl.Get return an error instead of using log.Fatalf
mostynb Jun 28, 2022
0ca05c9
[zstd] log zstd implementation mode (go/cgo) at startup
mostynb Jun 28, 2022
a004338
[zstd] add darwin-build.sh for quick iteration
mostynb Jun 28, 2022
f257568
[zstd] add cgo system test
mostynb Jun 28, 2022
245ee86
Build linux release binaries on ubuntu 18.04
mostynb Jun 28, 2022
9faa7de
[zstd] remove IOReadCloser TODO note
mostynb Jun 28, 2022
48f18d3
[zstd] fix infinite recursion in Close wrapper
mostynb Jun 28, 2022
1823529
[ci] don't attempt to cross compile now that we use cgo
mostynb Jul 16, 2022
67521a1
go mod tidy
mostynb Jul 16, 2022
1067ffd
Build without static setting
mostynb Jul 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 14 additions & 5 deletions .bazelci/presubmit.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@ tasks:
ubuntu1804:
platform: ubuntu1804
build_targets:
- "..."
- "//:bazel-remote"
- "//:bazel-remote-image"
test_targets:
- "..."
ubuntu2004:
platform: ubuntu2004
build_targets:
- "..."
- "//:bazel-remote"
- "//:bazel-remote-image"
test_targets:
- "..."
macos:
Expand Down Expand Up @@ -53,13 +55,20 @@ tasks:
- ".bazelci/buildkite-install-go.sh"
- "echo +++ Running go test with race detector"
- "PATH=$HOME/go/bin:$PATH go test -race ./..."
end_to_end_test:
end_to_end_test_go:
platform: ubuntu2004
name: "end-to-end test"
name: "end-to-end test - pure go zstd"
shell_commands:
- ".bazelci/buildkite-install-go.sh"
- "echo +++ Run end-to-end system test"
- "echo +++ Run end-to-end system test - pure go zstd"
- "PATH=$HOME/go/bin:$PATH timeout 30m .bazelci/system-test.sh"
end_to_end_test_cgo:
platform: ubuntu2004
name: "end-to-end test - cgo zstd"
shell_commands:
- ".bazelci/buildkite-install-go.sh"
- "echo +++ Run end-to-end system test - cgo zstd"
- "PATH=$HOME/go/bin:$PATH EXTRA_FLAGS=\"--zstd_implementation=cgo\" timeout 30m .bazelci/system-test.sh"
tls_tests:
platform: ubuntu2004
name: "TLS tests"
Expand Down
24 changes: 14 additions & 10 deletions .bazelci/system-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ HTTP_PORT=8082
min_acceptable_hit_rate=95
overall_result=success

EXTRA_FLAGS=""
EXTRA_FLAGS_DESC=""
[ -n "$EXTRA_FLAGS" ] && EXTRA_FLAGS_DESC="(with $EXTRA_FLAGS)"

summary=""

### Begin minio setup.
Expand Down Expand Up @@ -73,10 +77,10 @@ echo "${duration}s"
# Copy the binary somewhere known, so we can run it manually.
bazel run --run_under "cp -f " //:bazel-remote $(pwd)/

echo "Starting test cache"
echo "Starting test cache $EXTRA_FLAGS_DESC"
test_cache_dir=./bazel-remote-tmp-cache
rm -rf $test_cache_dir
./bazel-remote --max_size 1 --dir "$test_cache_dir" --http_address "0.0.0.0:$HTTP_PORT" \
./bazel-remote --max_size 1 --dir "$test_cache_dir" --http_address "0.0.0.0:$HTTP_PORT" $EXTRA_FLAGS \
--s3.endpoint 127.0.0.1:9000 \
--s3.bucket bazel-remote \
--s3.prefix files \
Expand All @@ -103,10 +107,10 @@ grep process http_cold

bazel clean 2> /dev/null

echo "Restarting test cache"
echo "Restarting test cache $EXTRA_FLAGS_DESC"
kill -9 $test_cache_pid
sleep 1
./bazel-remote --max_size 1 --dir $test_cache_dir --http_address "0.0.0.0:$HTTP_PORT" \
./bazel-remote --max_size 1 --dir $test_cache_dir --http_address "0.0.0.0:$HTTP_PORT" $EXTRA_FLAGS \
> log.stdout 2> log.stderr &
test_cache_pid=$!
echo "Test cache pid: $test_cache_pid"
Expand All @@ -130,11 +134,11 @@ result=$(awk -vhit_rate=$hit_rate -vmin=$min_acceptable_hit_rate 'BEGIN {if (hit
echo "hit rate: ${hit_rate}% (hits: $hits misses: $misses) => $result"
summary+="\n$testsection: hit rate: ${hit_rate}% (hits: $hits misses: $misses) => $result"

echo "Restarting test cache"
echo "Restarting test cache $EXTRA_FLAGS_DESC"
kill -9 $test_cache_pid
sleep 1
rm -rf $test_cache_dir
./bazel-remote --max_size 1 --dir $test_cache_dir --http_address "0.0.0.0:$HTTP_PORT" \
./bazel-remote --max_size 1 --dir $test_cache_dir --http_address "0.0.0.0:$HTTP_PORT" $EXTRA_FLAGS \
--s3.endpoint 127.0.0.1:9000 \
--s3.bucket bazel-remote \
--s3.prefix files \
Expand Down Expand Up @@ -168,11 +172,11 @@ result=$(awk -vhit_rate=$hit_rate -vmin=$min_acceptable_hit_rate 'BEGIN {if (hit
echo "hit rate: ${hit_rate}% (hits: $hits misses: $misses) => $result"
summary+="\n$testsection: hit rate: ${hit_rate}% (hits: $hits misses: $misses) => $result"

echo "Restarting test cache"
echo "Restarting test cache $EXTRA_FLAGS_DESC"
kill -9 $test_cache_pid
sleep 1
rm -rf $test_cache_dir
./bazel-remote --max_size 1 --dir $test_cache_dir --http_address "0.0.0.0:$HTTP_PORT" \
./bazel-remote --max_size 1 --dir $test_cache_dir --http_address "0.0.0.0:$HTTP_PORT" $EXTRA_FLAGS \
> log.stdout 2> log.stderr &
test_cache_pid=$!
echo "Test cache pid: $test_cache_pid"
Expand Down Expand Up @@ -209,11 +213,11 @@ result=$(awk -vhit_rate=$hit_rate -vmin=$min_acceptable_hit_rate 'BEGIN {if (hit
echo "hit rate: ${hit_rate}% (hits: $hits misses: $misses) => $result"
summary+="\n$testsection: hit rate: ${hit_rate}% (hits: $hits misses: $misses) => $result"

echo "Restarting test cache"
echo "Restarting test cache $EXTRA_FLAGS_DESC"
kill -9 $test_cache_pid
sleep 1
rm -rf $test_cache_dir
./bazel-remote --max_size 1 --dir $test_cache_dir --http_address "0.0.0.0:$HTTP_PORT" \
./bazel-remote --max_size 1 --dir $test_cache_dir --http_address "0.0.0.0:$HTTP_PORT" $EXTRA_FLAGS \
--s3.endpoint 127.0.0.1:9000 \
--s3.bucket bazel-remote \
--s3.prefix files \
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ on:
jobs:
linux:
name: create linux binaries
runs-on: ubuntu-20.04
runs-on: ubuntu-18.04
steps:
- name: checkout code
uses: actions/checkout@v2
Expand Down
12 changes: 6 additions & 6 deletions BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ go_library(
go_binary(
name = "bazel-remote",
embed = [":go_default_library"],
pure = "on",
static = "on",
pure = "off",
static = "off",
visibility = ["//visibility:public"],
x_defs = {"main.gitCommit": "{STABLE_GIT_COMMIT}"},
)
Expand All @@ -57,8 +57,8 @@ go_binary(
embed = [":go_default_library"],
goarch = "amd64",
goos = "linux",
pure = "on",
static = "on",
pure = "off",
static = "off",
visibility = ["//visibility:public"],
x_defs = {"main.gitCommit": "{STABLE_GIT_COMMIT}"},
)
Expand Down Expand Up @@ -104,7 +104,7 @@ go_image(
embed = [":go_default_library"],
goarch = "amd64",
goos = "linux",
pure = "on",
pure = "off",
static = "on",
visibility = ["//visibility:private"],
x_defs = {"main.gitCommit": "{STABLE_GIT_COMMIT}"},
Expand All @@ -115,7 +115,7 @@ go_image(
embed = [":go_default_library"],
goarch = "arm64",
goos = "linux",
pure = "on",
pure = "off",
static = "on",
visibility = ["//visibility:private"],
x_defs = {"main.gitCommit": "{STABLE_GIT_COMMIT}"},
Expand Down
3 changes: 3 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,9 @@ OPTIONS:
--storage_mode value Which format to store CAS blobs in. Must be one of
"zstd" or "uncompressed". (default: "zstd") [$BAZEL_REMOTE_STORAGE_MODE]

--zstd_implementation value ZSTD implementation to use. Must be one of
"go" or "cgo". (default: "go") [$BAZEL_REMOTE_ZSTD_IMPLEMENTATION]

--http_address value Address specification for the HTTP server listener,
formatted either as [host]:port for TCP or unix://path.sock for Unix
domain sockets. [$BAZEL_REMOTE_HTTP_ADDRESS]
Expand Down
1 change: 1 addition & 0 deletions cache/disk/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ go_library(
deps = [
"//cache:go_default_library",
"//cache/disk/casblob:go_default_library",
"//cache/disk/zstdimpl:go_default_library",
"//genproto/build/bazel/remote/execution/v2:go_default_library",
"//utils/tempfile:go_default_library",
"//utils/validate:go_default_library",
Expand Down
6 changes: 1 addition & 5 deletions cache/disk/casblob/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,7 @@ go_library(
srcs = ["casblob.go"],
importpath = "github.com/buchgr/bazel-remote/cache/disk/casblob",
visibility = ["//visibility:public"],
deps = [
"//utils/zstdpool:go_default_library",
"@com_github_klauspost_compress//zstd:go_default_library",
"@com_github_mostynb_zstdpool_syncpool//:go_default_library",
],
deps = ["//cache/disk/zstdimpl:go_default_library"],
)

go_test(
Expand Down
Loading