From 96a56735659ef0c8dc710dc4e4f6acccc2097522 Mon Sep 17 00:00:00 2001 From: Hleb Albau Date: Tue, 11 Dec 2018 18:03:27 +0300 Subject: [PATCH] #104 Make docker container based on nvidia-gpu image --- .circleci/config.yml | 123 +++++++++++------- .goreleaser.yml | 24 ---- Dockerfile | 91 ++++++++++--- app/app.go | 16 +-- cyberd/main.go | 2 +- entrypoint.sh | 7 +- start_script.sh | 2 +- {app => x}/rank/README.md | 4 +- {app => x}/rank/calculate.go | 0 {app => x}/rank/calculate_cpu.go | 0 {app => x}/rank/calculate_gpu.go | 0 {app => x}/rank/calculate_gpu_nop.go | 0 {app => x}/rank/cuda/cbdrank.h | 0 {app => x}/rank/cuda/rank.cu | 0 .../rank/cuda/test_cpu_gpu_determinism.go | 2 +- {app => x}/rank/cuda/test_rank.cu | 0 {app => x}/rank/cuda/types.h | 0 17 files changed, 165 insertions(+), 106 deletions(-) rename {app => x}/rank/README.md (97%) rename {app => x}/rank/calculate.go (100%) rename {app => x}/rank/calculate_cpu.go (100%) rename {app => x}/rank/calculate_gpu.go (100%) rename {app => x}/rank/calculate_gpu_nop.go (100%) rename {app => x}/rank/cuda/cbdrank.h (100%) rename {app => x}/rank/cuda/rank.cu (100%) rename {app => x}/rank/cuda/test_cpu_gpu_determinism.go (98%) rename {app => x}/rank/cuda/test_rank.cu (100%) rename {app => x}/rank/cuda/types.h (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 808f1ecd..1f2fb06f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,19 +22,20 @@ jobs: build_daemon_and_cli: working_directory: ~/cyberd docker: - - image: circleci/golang:1.11 + - image: circleci/golang:1.11 steps: - - checkout - - setup_remote_docker: - version: 17.11.0-ce - - run: - name: Build Cyberd Image - command: | - export GO111MODULE=on - go build -o daemon ./cyberd - go build -o cli ./cyberdcli - go build -o cyberdproxy ./proxy - go build -o cyberdclaim ./claim + - checkout + - setup_remote_docker: + version: 17.11.0-ce + - run: + name: Build And Test Project + command: | + export GO111MODULE=on + go build -o daemon ./cyberd + go build -o cli ./cyberdcli + go build -o cyberdproxy ./proxy + go build -o cyberdclaim ./claim + update_changelog: <<: *defaults @@ -43,7 +44,7 @@ jobs: version: 17.11.0-ce - checkout - run: - name: Build And Test Project + name: Update Changelog command: | cid=$(docker run -d -it ferrarimarco/github-changelog-generator:1.14.3 -u cybercongress -p cyberd --exclude-tags-regex .*[.].*[.].*[.].* --token $CHANGELOG_GITHUB_TOKEN ) docker attach $cid @@ -61,46 +62,64 @@ jobs: update_docs: <<: *defaults steps: - - checkout - - run: - name: Clone library repo - working_directory: ~/workdir - command: | - git clone -q --depth 1 https://${DOCS_GITHUB_TOKEN}@github.com/cybercongress/library.git - - deploy: - name: Update docs from cyberd repo - working_directory: ~/workdir/library - command: | - # We need to remove current files in folder to avoid duplicating renamed files - rm -rf ./wiki/docs/cyberd/* - cp -r ~/build/docs/. ./wiki/docs/cyberd/ - cp ~/build/CHANGELOG.md ./wiki/docs/cyberd/Changelog.md - cp ~/build/CONTRIBUTING.md ./wiki/docs/cyberd/Contributing.md - cp ~/build/ROADMAP.md ./wiki/docs/cyberd/Roadmap.md - git add -N wiki/docs/cyberd - diff=$(git diff wiki/docs/cyberd/) - if [[ -n "$diff" ]]; then - git config user.email "cybercongress42@gmail.com" - git config user.name "Cyber Admin" - git add wiki/docs/cyberd/ - git commit -m "Circle CI: Update Docs" - # Push quietly to prevent showing the token in log - git push -q https://${DOCS_GITHUB_TOKEN}@github.com/cybercongress/library.git master - fi + - checkout + - run: + name: Update Docs + working_directory: ~/workdir + command: | + git clone -q --depth 1 https://${DOCS_GITHUB_TOKEN}@github.com/cybercongress/library.git + - deploy: + name: Update docs from cyberd repo + working_directory: ~/workdir/library + command: | + # We need to remove current files in folder to avoid duplicating renamed files + rm -rf ./wiki/docs/cyberd/* + cp -r ~/build/docs/. ./wiki/docs/cyberd/ + cp ~/build/CHANGELOG.md ./wiki/docs/cyberd/Changelog.md + cp ~/build/CONTRIBUTING.md ./wiki/docs/cyberd/Contributing.md + cp ~/build/ROADMAP.md ./wiki/docs/cyberd/Roadmap.md + git add -N wiki/docs/cyberd + diff=$(git diff wiki/docs/cyberd/) + if [[ -n "$diff" ]]; then + git config user.email "cybercongress42@gmail.com" + git config user.name "Cyber Admin" + git add wiki/docs/cyberd/ + git commit -m "Circle CI: Update Docs" + # Push quietly to prevent showing the token in log + git push -q https://${DOCS_GITHUB_TOKEN}@github.com/cybercongress/library.git master + fi - release_cyberd_and_cyberdcli_binaries: + + build_and_deploy_images_to_dockerhub: + working_directory: ~/cyberd docker: - - image: circleci/golang:1.11 + - image: circleci/golang:1.11 + steps: + - checkout + - setup_remote_docker: + version: 17.11.0-ce + - run: + name: Build And Deploy Cyberd Image + command: | + docker build -t build/cyberd . + docker login -u $DOCKER_USER -p $DOCKER_PASS + docker tag build/cyberd cybernode/cyberd:$CIRCLE_TAG + docker push cybernode/cyberd:$CIRCLE_TAG + + + release_cyberdcli_binaries: + docker: + - image: circleci/golang:1.11 working_directory: /go/src/github.com/cybercongress/cyberd steps: - - checkout - - run: - name: Release Cyberd Binaries - command: | - export GO111MODULE=on - go build -o cli ./cyberdcli - sudo apt-get update && sudo apt-get install -y rpm - curl -sL https://git.io/goreleaser | bash + - checkout + - run: + name: Github Release Cyberd Binaries + command: | + export GO111MODULE=on + go build -o cyberdcli ./cyberdcli + sudo apt-get update && sudo apt-get install -y rpm + curl -sL https://git.io/goreleaser | bash workflows: version: 2 @@ -118,7 +137,11 @@ workflows: <<: *master_filter requires: - build_daemon_and_cli - - release_cyberd_and_cyberdcli_binaries: + - build_and_deploy_images_to_dockerhub: + <<: *release_filter + requires: + - build_daemon_and_cli + - release_cyberdcli_binaries: <<: *release_filter requires: - build_daemon_and_cli diff --git a/.goreleaser.yml b/.goreleaser.yml index 0d71638e..521d6541 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -11,30 +11,6 @@ builds: - windows goarch: - amd64 - - - main: ./proxy - binary: cyberdproxy - env: - - CGO_ENABLED=0 - - GO111MODULE=on - goos: - - linux - - darwin - - windows - goarch: - - amd64 - - - main: ./claim - binary: cyberdclaim - env: - - CGO_ENABLED=0 - - GO111MODULE=on - goos: - - linux - - darwin - - windows - goarch: - - amd64 checksum: diff --git a/Dockerfile b/Dockerfile index 6aee4907..9b01d12f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,90 @@ -FROM golang:1.11-alpine as builder +FROM nvidia/cuda:10.0-devel-ubuntu18.04 as build_stage -WORKDIR $GOPATH/src/github.com/cybercongress/cyberd -COPY ./ ./ +ENV GO_VERSION 1.11.2 +ENV GO_ARCH 'linux-amd64' +ENV GO_BIN_SHA '1dfe664fa3d8ad714bbd15a36627992effd150ddabd7523931f077b3926d736d' -RUN apk add --no-cache git -RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -ldflags="-w -s" -o /go/bin/cyberd ./cyberd -RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -ldflags="-w -s" -o /go/bin/cyberdcli ./cyberdcli -RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -ldflags="-w -s" -o /go/bin/cyberdproxy ./proxy +# Install required dev tools to compile cyberd +############################################################################### +RUN apt-get update && apt-get install -y --no-install-recommends wget git -FROM alpine:edge -RUN apk add --update ca-certificates -WORKDIR /root +# Install golang +############################################################################### +RUN url="https://golang.org/dl/go${GO_VERSION}.${GO_ARCH}.tar.gz" && \ + wget -O go.tgz "$url" && \ + echo "${GO_BIN_SHA} *go.tgz" | sha256sum -c - && \ + tar -C /usr/local -xzf go.tgz &&\ + rm go.tgz -COPY --from=builder /go/bin/cyberd /usr/bin/cyberd -COPY --from=builder /go/bin/cyberdcli /usr/bin/cyberdcli -COPY --from=builder /go/bin/cyberdproxy /usr/bin/cyberdproxy +ENV PATH="/usr/local/go/bin:$PATH" +RUN go version && nvcc --version + + +# Compile cuda kernel +############################################################################### +COPY . /sources +WORKDIR /sources/x/rank/cuda +RUN nvcc -fmad=false -shared -o libcbdrank.so rank.cu --compiler-options '-fPIC -frounding-math -fsignaling-nans' && \ + cp libcbdrank.so /usr/lib/ && cp cbdrank.h /usr/lib/ + + +# Compile cyberd +############################################################################### +WORKDIR /sources +RUN go build -tags cuda -o daemon ./cyberd +RUN go build -o cli ./cyberdcli +RUN go build -o daemon_proxy ./proxy + + +############################################################################### +# Create base image +############################################################################### +FROM nvidia/cuda:10.0-runtime-ubuntu18.04 + +ENV GO_VERSION 1.11.2 +ENV GO_ARCH 'linux-amd64' +ENV GO_BIN_SHA '1dfe664fa3d8ad714bbd15a36627992effd150ddabd7523931f077b3926d736d' + + +# Install required dev tools to install go +############################################################################### +RUN apt-get update && apt-get install -y --no-install-recommends wget curl -COPY start_script.sh start_script.sh -RUN chmod +x start_script.sh +# Install golang +############################################################################### +RUN url="https://golang.org/dl/go${GO_VERSION}.${GO_ARCH}.tar.gz" && \ + wget -O go.tgz "$url" && \ + echo "${GO_BIN_SHA} *go.tgz" | sha256sum -c - && \ + tar -C /usr/local -xzf go.tgz &&\ + rm go.tgz + + +# Copy compiled kernel and binaries +############################################################################### +COPY --from=build_stage /sources/daemon /usr/bin/cyberd +COPY --from=build_stage /sources/cli /usr/bin/cyberdcli +COPY --from=build_stage /sources/daemon_proxy /usr/bin/cyberdproxy + +COPY --from=build_stage /usr/lib/cbdrank.h /usr/lib/cbdrank.h +COPY --from=build_stage /usr/lib/libcbdrank.so /usr/lib/libcbdrank.so + + +# Copy configs and startup scripts +############################################################################### COPY ./testnet/genesis.json /genesis.json COPY ./testnet/config.toml /config.toml -EXPOSE 26656 26657 26660 - +COPY start_script.sh start_script.sh COPY entrypoint.sh /entrypoint.sh +RUN chmod +x start_script.sh RUN chmod +x /entrypoint.sh + +# Start +############################################################################### +EXPOSE 26656 26657 26660 ENTRYPOINT ["/entrypoint.sh"] CMD ["./start_script.sh"] diff --git a/app/app.go b/app/app.go index a92f53ff..64f8c2a9 100644 --- a/app/app.go +++ b/app/app.go @@ -15,13 +15,13 @@ import ( "github.com/cosmos/cosmos-sdk/x/stake" . "github.com/cybercongress/cyberd/app/bank" "github.com/cybercongress/cyberd/app/coin" - "github.com/cybercongress/cyberd/app/rank" . "github.com/cybercongress/cyberd/app/storage" cbd "github.com/cybercongress/cyberd/app/types" "github.com/cybercongress/cyberd/x/bandwidth" bw "github.com/cybercongress/cyberd/x/bandwidth/types" "github.com/cybercongress/cyberd/x/link" "github.com/cybercongress/cyberd/x/mint" + "github.com/cybercongress/cyberd/x/rank" abci "github.com/tendermint/tendermint/abci/types" cmn "github.com/tendermint/tendermint/libs/common" dbm "github.com/tendermint/tendermint/libs/db" @@ -35,7 +35,7 @@ import ( const ( APP = "cyberd" appName = "CyberdApp" - DefaultKeyPass = "12345678" + DefaultKeyPass = "12345678" //todo remove from here ) // default home directories for expected binaries @@ -72,10 +72,10 @@ type CyberdApp struct { txDecoder sdk.TxDecoder // bandwidth - bandwidthHandler bw.BandwidthHandler - msgBandwidthCost bw.MsgBandwidthCost - maxAccBandwidth bw.MaxAccBandwidth - curBlockSpentBandwidth uint64 //resets every block + bandwidthHandler bw.BandwidthHandler + msgBandwidthCost bw.MsgBandwidthCost + maxAccBandwidth bw.MaxAccBandwidth + curBlockSpentBandwidth uint64 //resets every block lastTotalSpentBandwidth uint64 //resets every bandwidth price adjustment interval currentCreditPrice float64 @@ -100,8 +100,8 @@ type CyberdApp struct { persistStorages CyberdPersistentStorages memStorage *InMemoryStorage - latestRankHash []byte - latestBlockHeight int64 + latestRankHash []byte + latestBlockHeight int64 computeUnit rank.ComputeUnit } diff --git a/cyberd/main.go b/cyberd/main.go index 24c55b91..9977fbfc 100644 --- a/cyberd/main.go +++ b/cyberd/main.go @@ -5,9 +5,9 @@ import ( "github.com/cosmos/cosmos-sdk/baseapp" "github.com/cosmos/cosmos-sdk/server" "github.com/cybercongress/cyberd/app" - "github.com/cybercongress/cyberd/app/rank" initCyberd "github.com/cybercongress/cyberd/cyberd/init" "github.com/cybercongress/cyberd/cyberd/rpc" + "github.com/cybercongress/cyberd/x/rank" "github.com/spf13/cobra" "github.com/spf13/viper" abci "github.com/tendermint/tendermint/abci/types" diff --git a/entrypoint.sh b/entrypoint.sh index 6f2b0a02..3cbf3b78 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -3,6 +3,7 @@ if [ ! -f "/root/.cyberd/config/genesis.json" ] then cp /genesis.json /root/.cyberd/config/ + cat /root/.cyberd/config/genesis.json fi if [ ! -f "/root/.cyberd/config/config.toml" ] @@ -10,8 +11,10 @@ then cp /config.toml /root/.cyberd/config/ fi -cat /root/.cyberd/config/genesis.json -cyberd init +if [ ! -f "/root/.cyberd/config/node_key.json" ] +then + cyberd init +fi exec "$@" \ No newline at end of file diff --git a/start_script.sh b/start_script.sh index a55109dc..f2b7abe0 100644 --- a/start_script.sh +++ b/start_script.sh @@ -1,7 +1,7 @@ #!/bin/sh # Start the first process -cyberd start --compute-rank-on-gpu=false . & +cyberd start . & status=$? if [ $status -ne 0 ]; then echo "Failed to start cyberd: $status" diff --git a/app/rank/README.md b/x/rank/README.md similarity index 97% rename from app/rank/README.md rename to x/rank/README.md index 63581db7..6ce3e074 100644 --- a/app/rank/README.md +++ b/x/rank/README.md @@ -7,7 +7,7 @@ Next compile **cbdrank** lib, copy it to `/usr/lib/` folder: ```bash # project root -cd app/rank/cuda +cd x/rank/cuda nvcc -fmad=false -shared -o libcbdrank.so rank.cu --compiler-options '-fPIC -frounding-math -fsignaling-nans' sudo cp libcbdrank.so /usr/lib/ sudo cp cbdrank.h /usr/lib/ @@ -26,7 +26,7 @@ go build -tags cuda -o daemon ./cyberd To test GPU and CPU rank computing determinism run: ```bash # project root -cd app/rank/cuda +cd x/rank/cuda nvcc -fmad=false -shared -o libcbdrank.so rank.cu --compiler-options '-fPIC -frounding-math -fsignaling-nans' sudo cp libcbdrank.so /usr/lib/ sudo cp cbdrank.h /usr/lib/ diff --git a/app/rank/calculate.go b/x/rank/calculate.go similarity index 100% rename from app/rank/calculate.go rename to x/rank/calculate.go diff --git a/app/rank/calculate_cpu.go b/x/rank/calculate_cpu.go similarity index 100% rename from app/rank/calculate_cpu.go rename to x/rank/calculate_cpu.go diff --git a/app/rank/calculate_gpu.go b/x/rank/calculate_gpu.go similarity index 100% rename from app/rank/calculate_gpu.go rename to x/rank/calculate_gpu.go diff --git a/app/rank/calculate_gpu_nop.go b/x/rank/calculate_gpu_nop.go similarity index 100% rename from app/rank/calculate_gpu_nop.go rename to x/rank/calculate_gpu_nop.go diff --git a/app/rank/cuda/cbdrank.h b/x/rank/cuda/cbdrank.h similarity index 100% rename from app/rank/cuda/cbdrank.h rename to x/rank/cuda/cbdrank.h diff --git a/app/rank/cuda/rank.cu b/x/rank/cuda/rank.cu similarity index 100% rename from app/rank/cuda/rank.cu rename to x/rank/cuda/rank.cu diff --git a/app/rank/cuda/test_cpu_gpu_determinism.go b/x/rank/cuda/test_cpu_gpu_determinism.go similarity index 98% rename from app/rank/cuda/test_cpu_gpu_determinism.go rename to x/rank/cuda/test_cpu_gpu_determinism.go index 17dafdd3..8a2d5d74 100644 --- a/app/rank/cuda/test_cpu_gpu_determinism.go +++ b/x/rank/cuda/test_cpu_gpu_determinism.go @@ -2,9 +2,9 @@ package main import ( "fmt" - cpurank "github.com/cybercongress/cyberd/app/rank" . "github.com/cybercongress/cyberd/app/storage" . "github.com/cybercongress/cyberd/app/types" + cpurank "github.com/cybercongress/cyberd/x/rank" "github.com/tendermint/tendermint/libs/log" ) diff --git a/app/rank/cuda/test_rank.cu b/x/rank/cuda/test_rank.cu similarity index 100% rename from app/rank/cuda/test_rank.cu rename to x/rank/cuda/test_rank.cu diff --git a/app/rank/cuda/types.h b/x/rank/cuda/types.h similarity index 100% rename from app/rank/cuda/types.h rename to x/rank/cuda/types.h