From 595d029bef723dfa75de87ca0f9588ae57cf6a9a Mon Sep 17 00:00:00 2001 From: VM <112189277+sysvm@users.noreply.github.com> Date: Fri, 3 Feb 2023 22:17:29 +0800 Subject: [PATCH] ci: add commit lint, code lint and unit test ci files (#67) --- .github/coverage.yml | 144 ++++++++++++++++++++++++++++ .github/workflows/build-test.yml | 88 +++++++++++++++++ .github/workflows/code-lint.yml | 86 +++++++++++++++++ .github/workflows/commit-lint.yml | 81 ++++++++++++++++ .github/workflows/go.yml | 28 ------ .github/workflows/proto-lint.yml | 65 +++++++++++++ .github/workflows/uint-test.yml | 88 +++++++++++++++++ .golangci.yml | 31 ++++++ Dockerfile | 45 +++++++++ Makefile | 11 ++- go.mod | 1 - service/syncer/server.go | 1 + test/e2e/piecestore/meomory_test.go | 5 - test/e2e/piecestore/s3_test.go | 5 - 14 files changed, 636 insertions(+), 43 deletions(-) create mode 100644 .github/coverage.yml create mode 100644 .github/workflows/build-test.yml create mode 100644 .github/workflows/code-lint.yml create mode 100644 .github/workflows/commit-lint.yml delete mode 100644 .github/workflows/go.yml create mode 100644 .github/workflows/proto-lint.yml create mode 100644 .github/workflows/uint-test.yml create mode 100644 .golangci.yml create mode 100644 Dockerfile diff --git a/.github/coverage.yml b/.github/coverage.yml new file mode 100644 index 000000000..e7d40b54c --- /dev/null +++ b/.github/coverage.yml @@ -0,0 +1,144 @@ +name: Test Coverage + +on: + push: + branches: + - main + - develop + + pull_request: + branches: + - main + - develop + +jobs: + split-test-files: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Create a file with all the pkgs + run: go list ./... > pkgs.txt + - name: Split pkgs into 4 files + run: split -d -n l/4 pkgs.txt pkgs.txt.part. + # cache multiple + - uses: actions/upload-artifact@v3 + with: + name: "${{ github.sha }}-00" + path: ./pkgs.txt.part.00 + - uses: actions/upload-artifact@v3 + with: + name: "${{ github.sha }}-01" + path: ./pkgs.txt.part.01 + - uses: actions/upload-artifact@v3 + with: + name: "${{ github.sha }}-02" + path: ./pkgs.txt.part.02 + - uses: actions/upload-artifact@v3 + with: + name: "${{ github.sha }}-03" + path: ./pkgs.txt.part.03 + + build-linux: + name: Build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + goarch: ["amd64"] + go-version: [1.19.x] + timeout-minutes: 5 + steps: + - uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - name: Buf install + uses: bufbuild/buf-setup-action@v1.11.0 + - name: Buf generate + run: | + buf generate + - uses: actions/checkout@v3 + - uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + **/**.go + go.mod + go.sum + - name: install + run: GOOS=linux GOARCH=${{ matrix.goarch }} make build + if: "env.GIT_DIFF != ''" + + tests: + runs-on: ubuntu-latest + needs: split-test-files + strategy: + fail-fast: false + matrix: + part: ["00", "01", "02", "03"] + steps: + - uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - uses: actions/checkout@v3 + - name: Buf install + uses: bufbuild/buf-setup-action@v1.11.0 + - name: Buf generate + run: | + buf generate + - uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + **/**.go + go.mod + go.sum + - uses: actions/download-artifact@v3 + with: + name: "${{ github.sha }}-${{ matrix.part }}" + if: env.GIT_DIFF + - name: test & coverage report creation + run: | + cat pkgs.txt.part.${{ matrix.part }} | xargs go test -mod=readonly -timeout 8m -race -coverprofile=${{ matrix.part }}profile.out -covermode=atomic + if: env.GIT_DIFF + - uses: actions/upload-artifact@v3 + with: + name: "${{ github.sha }}-${{ matrix.part }}-coverage" + path: ./${{ matrix.part }}profile.out + + upload-coverage-report: + runs-on: ubuntu-latest + needs: tests + steps: + - uses: actions/checkout@v3 + - name: Buf install + uses: bufbuild/buf-setup-action@v1.11.0 + - name: Buf generate + run: | + buf generate + - uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + **/**.go + go.mod + go.sum + - uses: actions/download-artifact@v3 + with: + name: "${{ github.sha }}-00-coverage" + if: env.GIT_DIFF + - uses: actions/download-artifact@v3 + with: + name: "${{ github.sha }}-01-coverage" + if: env.GIT_DIFF + - uses: actions/download-artifact@v3 + with: + name: "${{ github.sha }}-02-coverage" + if: env.GIT_DIFF + - uses: actions/download-artifact@v3 + with: + name: "${{ github.sha }}-03-coverage" + if: env.GIT_DIFF + - run: | + cat ./*profile.out | grep -v "mode: atomic" >> coverage.txt + if: env.GIT_DIFF + - uses: codecov/codecov-action@v3 + with: + file: ./coverage.txt + if: env.GIT_DIFF \ No newline at end of file diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml new file mode 100644 index 000000000..5c39cb384 --- /dev/null +++ b/.github/workflows/build-test.yml @@ -0,0 +1,88 @@ +name: Build Test + +on: + push: + branches: + - main + - develop + + pull_request: + branches: + - main + - develop + +jobs: + build-test: + strategy: + matrix: + go-version: [1.19.x] + os: [ubuntu-20.04] + runs-on: ${{ matrix.os }} + steps: + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + + - name: Checkout code + uses: actions/checkout@v3 + + - uses: actions/cache@v3 + with: + # In order: + # * Module download cache + # * Build cache (Linux) + # * Build cache (Mac) + # * Build cache (Windows) + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - uses: ory/build-buf-action@v0 + with: + bufVersion: v1.11.0 + protocPlugins: + go@v1.28.0 go-grpc@v1.2.0 + bufArgs: + generate + + - name: Test Build + run: | + bash build.sh + + build-test-success: + needs: build-test + if: ${{ success() }} + runs-on: ubuntu-latest + steps: + - name: Notify Slack on success + uses: rtCamp/action-slack-notify@v2.2.0 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + SLACK_CHANNEL: greenfield_storage_ci + SLACK_USERNAME: SP Build Test + SLACK_ICON_EMOJI: ':heavy_check_mark:' + SLACK_COLOR: good + SLACK_MESSAGE: Greenfield SP build test passed + SLACK_FOOTER: '' + + build-test-failure: + needs: build-test + if: ${{ failure() }} + runs-on: ubuntu-latest + steps: + - name: Notify Slack on failure + uses: rtCamp/action-slack-notify@v2.2.0 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + SLACK_CHANNEL: greenfield_storage_ci + SLACK_USERNAME: SP Build Test + SLACK_ICON_EMOJI: ':x:' + SLACK_COLOR: danger + SLACK_MESSAGE: Greenfield SP build test failed + SLACK_FOOTER: '' \ No newline at end of file diff --git a/.github/workflows/code-lint.yml b/.github/workflows/code-lint.yml new file mode 100644 index 000000000..9b814c67f --- /dev/null +++ b/.github/workflows/code-lint.yml @@ -0,0 +1,86 @@ +name: Golang Code Lint +# Lint runs golangci-lint over the entire Tendermint repository +# This workflow is run on every pull request and push to master +# The `golangci` job will pass without running if no *.{go, mod, sum} files have been modified. +on: + push: + branches: + - main + - develop + + pull_request: + branches: + - main + - develop + +jobs: + golangci: + name: golangci-lint + runs-on: ubuntu-latest + timeout-minutes: 8 + strategy: + matrix: + go-version: [1.19.x] + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + + - uses: ory/build-buf-action@v0 + with: + bufVersion: v1.11.0 + protocPlugins: + go@v1.28.0 go-grpc@v1.2.0 + bufArgs: + generate + + - uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + **/**.go + go.mod + go.sum + + - uses: golangci/golangci-lint-action@v3 + with: + # Required: the version of golangci-lint is required and + # must be specified without patch version: we always use the + # latest patch version. + version: v1.50.1 + args: --timeout 10m + github-token: ${{ secrets.github_token }} + if: env.GIT_DIFF + + golang-ci-lint-success: + needs: golangci + if: ${{ success() }} + runs-on: ubuntu-latest + steps: + - name: Notify Slack on success + uses: rtCamp/action-slack-notify@v2.2.0 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + SLACK_CHANNEL: greenfield_storage_ci + SLACK_USERNAME: SP Golang CI Lint + SLACK_ICON_EMOJI: ':heavy_check_mark:' + SLACK_COLOR: good + SLACK_MESSAGE: Greenfield golang ci lint passed + SLACK_FOOTER: '' + + golang-ci-lint-failure: + needs: golangci + if: ${{ failure() }} + runs-on: ubuntu-latest + steps: + - name: Notify Slack on failure + uses: rtCamp/action-slack-notify@v2.2.0 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + SLACK_CHANNEL: greenfield_storage_ci + SLACK_USERNAME: SP Golang CI Lint + SLACK_ICON_EMOJI: ':x:' + SLACK_COLOR: danger + SLACK_MESSAGE: Greenfield golang ci lint failed + SLACK_FOOTER: '' \ No newline at end of file diff --git a/.github/workflows/commit-lint.yml b/.github/workflows/commit-lint.yml new file mode 100644 index 000000000..cecad3b7a --- /dev/null +++ b/.github/workflows/commit-lint.yml @@ -0,0 +1,81 @@ +name: Lint Commit Messages + +on: + push: + branches: + - main + - develop + + pull_request: + branches: + - main + - develop + +jobs: + commit-lint: + strategy: + matrix: + node-version: [14.x] + os: [ubuntu-20.04] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + + - uses: actions/cache@v3 + with: + path: | + ~/.npm + **/node_modules + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + + - name: Install Deps + run: | + npm install -g commitlint-plugin-function-rules @commitlint/cli + npm install --save-dev commitlint-plugin-function-rules @commitlint/cli + + - uses: wagoid/commitlint-github-action@v5 + id: commitlint + env: + NODE_PATH: ${{ github.workspace }}/node_modules + with: + configFile: /github/workspace/.github/commitlint.config.js + + commit-lint-success: + needs: commit-lint + if: ${{ success() }} + runs-on: ubuntu-latest + steps: + - name: Notify Slack on success + uses: rtCamp/action-slack-notify@v2.2.0 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + SLACK_CHANNEL: greenfield_storage_ci + SLACK_USERNAME: SP Git Lint + SLACK_ICON_EMOJI: ':heavy_check_mark:' + SLACK_COLOR: good + SLACK_MESSAGE: Greenfield SP git commit lint passed + SLACK_FOOTER: '' + + commit-lint-failure: + needs: commit-lint + if: ${{ failure() }} + runs-on: ubuntu-latest + steps: + - name: Notify Slack on failure + uses: rtCamp/action-slack-notify@v2.2.0 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + SLACK_CHANNEL: greenfield_storage_ci + SLACK_USERNAME: SP Git Lint + SLACK_ICON_EMOJI: ':x:' + SLACK_COLOR: danger + SLACK_MESSAGE: Greenfield SP git commit lint failed + SLACK_FOOTER: '' \ No newline at end of file diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml deleted file mode 100644 index 6e04abdda..000000000 --- a/.github/workflows/go.yml +++ /dev/null @@ -1,28 +0,0 @@ -# This workflow will build a golang project -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go - -name: Go - -on: - push: - branches: [ "main, develop" ] - pull_request: - branches: [ "main, develop" ] - -jobs: - - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: 1.19 - - - name: Build - run: go build -v ./... - - - name: Test - run: go test -v ./... diff --git a/.github/workflows/proto-lint.yml b/.github/workflows/proto-lint.yml new file mode 100644 index 000000000..c63043b56 --- /dev/null +++ b/.github/workflows/proto-lint.yml @@ -0,0 +1,65 @@ +name: Protobuf Lint + +on: + push: + branches: + - main + - develop + paths: + - 'proto/**' + + pull_request: + branches: + - main + - develop + paths: + - 'proto/**' + +jobs: + proto-lint: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v3 + + - uses: bufbuild/buf-setup-action@v1.11.0 + + - uses: bufbuild/buf-lint-action@v1 + with: + input: 'proto' + + - uses: bufbuild/buf-breaking-action@v1 + with: + against: "https//github.com/bnb-chain/greenfield-storage-provider" + + proto-lint-success: + needs: proto-lint + if: ${{ success() }} + runs-on: ubuntu-latest + steps: + - name: Notify Slack on success + uses: rtCamp/action-slack-notify@v2.2.0 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + SLACK_CHANNEL: greenfield_storage_ci + SLACK_USERNAME: SP Protobuf Lint + SLACK_ICON_EMOJI: ':heavy_check_mark:' + SLACK_COLOR: good + SLACK_MESSAGE: Greenfield SP protobuf lint passed + SLACK_FOOTER: '' + + proto-lint-failure: + needs: proto-lint + if: ${{ failure() }} + runs-on: ubuntu-latest + steps: + - name: Notify Slack on failure + uses: rtCamp/action-slack-notify@v2.2.0 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + SLACK_CHANNEL: greenfield_storage_ci + SLACK_USERNAME: SP Protobuf Lint + SLACK_ICON_EMOJI: ':x:' + SLACK_COLOR: danger + SLACK_MESSAGE: Greenfield SP protobuf lint failed + SLACK_FOOTER: '' \ No newline at end of file diff --git a/.github/workflows/uint-test.yml b/.github/workflows/uint-test.yml new file mode 100644 index 000000000..7f7002c55 --- /dev/null +++ b/.github/workflows/uint-test.yml @@ -0,0 +1,88 @@ +name: Unit Test + +on: + push: + branches: + - main + - develop + + pull_request: + branches: + - main + - develop + +jobs: + unit-test: + strategy: + matrix: + go-version: [1.19.x] + os: [ubuntu-20.04] + runs-on: ${{ matrix.os }} + steps: + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + + - name: Checkout code + uses: actions/checkout@v3 + + - uses: actions/cache@v3 + with: + # In order: + # * Module download cache + # * Build cache (Linux) + # * Build cache (Mac) + # * Build cache (Windows) + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - uses: ory/build-buf-action@v0 + with: + bufVersion: v1.11.0 + protocPlugins: + go@v1.28.0 go-grpc@v1.2.0 + bufArgs: + generate + + - name: Unit Test + run: | + make test + + unit-test-success: + needs: unit-test + if: ${{ success() }} + runs-on: ubuntu-latest + steps: + - name: Notify Slack on success + uses: rtCamp/action-slack-notify@v2.2.0 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + SLACK_CHANNEL: greenfield_storage_ci + SLACK_USERNAME: SP Unit Test + SLACK_ICON_EMOJI: ':heavy_check_mark:' + SLACK_COLOR: good + SLACK_MESSAGE: Greenfield SP unit test passed + SLACK_FOOTER: '' + + unit-test-failure: + needs: unit-test + if: ${{ failure() }} + runs-on: ubuntu-latest + steps: + - name: Notify Slack on failure + uses: rtCamp/action-slack-notify@v2.2.0 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + SLACK_CHANNEL: greenfield_storage_ci + SLACK_USERNAME: SP Unit Test + SLACK_ICON_EMOJI: ':x:' + SLACK_COLOR: danger + SLACK_MESSAGE: Greenfield SP unit test failed + SLACK_FOOTER: '' \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 000000000..ba67b5553 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,31 @@ +# This file configures github.com/golangci/golangci-lint. + +run: + concurrency: 4 + timeout: 5m + tests: true + # default is true. Enables skipping of directories: + # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ + skip-dirs-use-default: true + +linters: + disable-all: true + enable: + - deadcode + - goconst + - goimports + - gosimple + - govet + - ineffassign + - misspell + - staticcheck + - unconvert + - unused + - varcheck + +linters-settings: + gofmt: + simplify: true + goconst: + min-len: 3 # minimum length of string constant + min-occurrences: 6 # minimum number of occurrences diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..0b8a73bd9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ +# Support setting various labels on the final image +ARG COMMIT="" +ARG VERSION="" +ARG BUILDNUM="" + +# Build storage provider in a stock Go builder container +FROM golang:1.19-alpine as builder + +RUN apk add --no-cache make git bash + +ADD . /storageprovider + +ENV CGO_ENABLED=0 +ENV GO111MODULE=on + +RUN cd /storageprovider && make build + +# Pull storage provider into a second stage deploy alpine container +FROM alpine:3.16.0 + +ARG SP_USER=sp +ARG SP_USER_UID=1000 +ARG SP_USER_GID=1000 + +ENV PACKAGES ca-certificates bash curl libstdc++ +ENV WORKDIR=/server + +RUN apk add --no-cache $PACKAGES \ + && rm -rf /var/cache/apk/* \ + && addgroup -g ${SP_USER_GID} ${SP_USER} \ + && adduser -u ${SP_USER_UID} -G ${SP_USER} --shell /sbin/nologin --no-create-home -D ${SP_USER} \ + && addgroup ${SP_USER} tty \ + && sed -i -e "s/bin\/sh/bin\/bash/" /etc/passwd + +RUN echo "[ ! -z \"\$TERM\" -a -r /etc/motd ] && cat /etc/motd" >> /etc/bash/bashrc + +WORKDIR ${WORKDIR} + +COPY --from=builder /storageprovider/build/bin/storage_provider ${WORKDIR}/ +RUN chown -R ${SP_USER_UID}:${SP_USER_GID} ${WORKDIR} +USER ${SP_USER_UID}:${SP_USER_GID} + +EXPOSE 9033 9133 9233 9333 9433 9533 + +ENTRYPOINT ["/server/storage_provider"] diff --git a/Makefile b/Makefile index b2a7452bd..8645f8d03 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,13 @@ SHELL := /bin/bash -.PHONY: all check format vet generate build tidy clean +.PHONY: all format vet generate build tidy test clean help: @echo "Please use \`make \` where is one of" - @echo " check to do static check" + @echo " vet to do static check" @echo " build to create bin directory and build" @echo " generate to generate code" -check: vet - format: gofmt -w -l . @@ -26,5 +24,10 @@ tidy: go mod tidy go mod verify +# only run unit test, exclude e2e tests +test: + go test `go list ./... | grep -v /test/` + # go test -cover ./... + clean: rm -rf ./pkg/types/v1/*.pb.go && rm -rf ./service/types/v1/*.pb.go diff --git a/go.mod b/go.mod index 4b03fbf32..d4da16d5d 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/bytedance/gopkg v0.0.0-20221122125632-68358b8ecec6 github.com/cosmos/cosmos-proto v1.0.0-beta.1 github.com/golang/mock v1.6.0 - github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 github.com/json-iterator/go v1.1.12 github.com/klauspost/reedsolomon v1.11.3 diff --git a/service/syncer/server.go b/service/syncer/server.go index f279034b7..072864c60 100644 --- a/service/syncer/server.go +++ b/service/syncer/server.go @@ -43,6 +43,7 @@ func NewSyncerService(config *SyncerConfig) (*Syncer, error) { return s, nil } +// initClient func (s *Syncer) initClient() error { store, err := client.NewStoreClient(s.cfg.PieceConfig) if err != nil { diff --git a/test/e2e/piecestore/meomory_test.go b/test/e2e/piecestore/meomory_test.go index 945501a67..66b19315a 100644 --- a/test/e2e/piecestore/meomory_test.go +++ b/test/e2e/piecestore/meomory_test.go @@ -6,11 +6,6 @@ import ( "os" "testing" -<<<<<<< HEAD - "github.com/bnb-chain/greenfield-storage-provider/util/log" - -======= ->>>>>>> develop "github.com/stretchr/testify/assert" "github.com/bnb-chain/greenfield-storage-provider/util/log" diff --git a/test/e2e/piecestore/s3_test.go b/test/e2e/piecestore/s3_test.go index c9fb11853..4cfdb6a82 100644 --- a/test/e2e/piecestore/s3_test.go +++ b/test/e2e/piecestore/s3_test.go @@ -6,11 +6,6 @@ import ( "os" "testing" -<<<<<<< HEAD - "github.com/bnb-chain/greenfield-storage-provider/util/log" - -======= ->>>>>>> develop "github.com/stretchr/testify/assert" "github.com/bnb-chain/greenfield-storage-provider/util/log"