diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..bc8edf0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,34 @@ +--- +name: Bug Report +about: Report a bug encountered while using openebs exporter +labels: kind/bug + +--- + + + +**Describe the bug:** A clear and concise description of what the bug is. + +**Expected behaviour:** A concise description of what you expected to happen + +**Steps to reproduce the bug:** +Steps to reproduce the bug should be clear and easily reproducible to help people gain an understanding of the problem + +**The output of the following commands will help us better understand what's going on**: + + +* `kubectl get pods -n --show-labels` +* `kubectl logs -n ` + +**Anything else we need to know?:** +Add any other context about the problem here. + +**Environment details:** +- OpenEBS version (use `kubectl get po -n openebs --show-labels`): +- Kubernetes version (use `kubectl version`): +- Cloud provider or hardware configuration: +- OS (e.g: `cat /etc/os-release`): +- kernel (e.g: `uname -a`): +- others: diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md new file mode 100644 index 0000000..cbda028 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -0,0 +1,26 @@ +--- +name: Feature request +about: Suggest an idea to improve openebs exporter +labels: Enhancement + +--- + +**Describe the problem/challenge you have** +[A description of the current limitation/problem/challenge that you are experiencing.] + + +**Describe the solution you'd like** +[A clear and concise description of what you want to happen.] + + +**Anything else you would like to add:** +[Miscellaneous information that will assist in solving the issue.] + + +**Environment:** +- OpenEBS version (use `kubectl get po -n openebs --show-labels`): +- Kubernetes version (use `kubectl version`): +- Cloud provider or hardware configuration: +- OS (e.g: `cat /etc/os-release`): +- kernel (e.g: `uname -a`): +- others: diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..c393472 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,41 @@ +## Pull Request template + +**Why is this PR required? What issue does it fix?**: + +**What this PR does?**: + +**Does this PR require any upgrade changes?**: + +**If the changes in this PR are manually verified, list down the scenarios covered:**: + +**Any additional information for your reviewer?** : +_Mention if this PR is part of any design or a continuation of previous PRs_ + + +**Checklist:** +- [ ] Fixes # +- [ ] PR Title follows the convention of `(): ` +- [ ] Has the change log section been updated? +- [ ] Commit has unit tests +- [ ] Commit has integration tests +- [ ] (Optional) Are upgrade changes included in this PR? If not, mention the issue/PR to track: +- [ ] (Optional) If documentation changes are required, which issue on https://github.com/openebs/openebs-docs is used to track them: + + +**PLEASE REMOVE BELOW INFORMATION BEFORE SUBMITTING** + +The PR title message must follow convention: + `(): `. + +Where: + Most common types are: + * `feat` - for new features, not a new feature for build script + * `fix` - for bug fixes or improvements, not a fix for build script + * `chore` - changes not related to production code + * `docs` - changes related to documentation + * `style` - formatting, missing semi colons, linting fix etc; no significant production code changes + * `test` - adding missing tests, refactoring tests; no production code change + * `refactor` - refactoring production code, eg. renaming a variable or function name, there should not be any significant production code changes + * `cherry-pick` - if PR is merged in master branch and raised to release branch(like v0.4.x) + +IMPORTANT: Please review the [CONTRIBUTING.md](../CONTRIBUTING.md) file for detailed contributing guidelines. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7a1e340..b7b37f1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,11 +48,14 @@ jobs: - name: verify license run: make check-license - - name: verify dependencies - run: make deps - - name: verify tests run: make test + + - name: Verify corrections + run: make verify-src + + - name: Upload Coverage Report + uses: codecov/codecov-action@v1 exporter: runs-on: ubuntu-latest @@ -87,8 +90,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v1 with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build & Push Image env: diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 6b585fa..00ea1a3 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -54,12 +54,15 @@ jobs: - name: verify license run: make check-license - - name: verify dependencies - run: make deps - - name: verify tests run: make test + - name: Verify corrections + run: make verify-src + + - name: Upload Coverage Report + uses: codecov/codecov-action@v1 + exporter: runs-on: ubuntu-latest needs: ['lint', 'test'] diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4ad90e4..f46beb1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,8 +46,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v1 with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build & Push Image env: diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..8d0dfc7 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,46 @@ +sudo: required + +dist: xenial + +env: + global: + - CAN_FAIL=false + - RELEASE_TAG_DOWNSTREAM=0 + - RELEASE_TAG="$TRAVIS_TAG" + - BRANCH="$TRAVIS_BRANCH" + +services: + - docker + +language: go + +cache: + directories: + - $HOME/.cache/go-build + +go: + - 1.14.7 + +addons: + apt: + update: true + +install: + - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + - sudo apt-get update -qq + +before_script: + - make test + - make verify-src + +script: + - make exporter-image EXPORTER_IMAGE=m-exporter-amd64 + +after_success: + - make push EXPORTER_IMAGE=m-exporter-amd64 + +notifications: + email: + recipients: + - kiran.mova@mayadata.io + - shubham.bajpai@mayadata.io diff --git a/Makefile b/Makefile index 6e7dbe6..bc28185 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,12 @@ export XC_ARCH ARCH:=${XC_OS}_${XC_ARCH} export ARCH +# list only the source code directories +PACKAGES = $(shell go list ./... | grep -v 'vendor\|pkg/client/generated\|tests') + +# list only the integration tests code directories +PACKAGES_IT = $(shell go list ./... | grep -v 'vendor\|pkg/client/generated' | grep 'tests') + # Specify the name for the binaries EXPORTER=exporter @@ -78,12 +84,33 @@ deps: @go mod verify .PHONY: test -test: - go test ./... +test: format vet + @echo "--> Running go test"; + $(PWD)/build/test.sh ${XC_ARCH} + +.PHONY: testv +testv: format + @echo "--> Running go test verbose" ; + @go test -v $(PACKAGES) + +.PHONY: format +format: + @echo "--> Running go fmt" + @go fmt $(PACKAGES) $(PACKAGES_IT) + +# -composite: avoid "literal copies lock value from fakePtr" +.PHONY: vet +vet: + @echo "--> Running go vet" + @go list ./... | grep -v "./vendor/*" | xargs go vet -composites + +.PHONY: verify-src +verify-src: + @echo "--> Checking for git changes post running tests"; + $(PWD)/build/check-diff.sh "format" # Specify the name of the docker repo for amd64 -EXPORTER_REPO_NAME="exporter" -EXPORTER_IMAGE_NAME="m-exporter" +EXPORTER_IMAGE?="m-exporter" ifeq (${IMAGE_TAG}, ) IMAGE_TAG = ci @@ -112,20 +139,20 @@ export DBUILD_ARGS=--build-arg BASE_IMAGE=$(CSTOR_BASE_IMAGE) --build-arg DBUILD exporter-image: exporter @echo "-----------------------------------------------" @echo "--> ${EXPORTER} image " - @echo "${IMAGE_ORG}/${EXPORTER_IMAGE_NAME}:${IMAGE_TAG}" + @echo "${IMAGE_ORG}/${EXPORTER_IMAGE}:${IMAGE_TAG}" @echo "-----------------------------------------------" @cp bin/${EXPORTER}/${EXPORTER} build/${EXPORTER} @cd build/${EXPORTER} && \ - sudo docker build -t "${IMAGE_ORG}/${EXPORTER_IMAGE_NAME}:${IMAGE_TAG}" ${DBUILD_ARGS} . + sudo docker build -t "${IMAGE_ORG}/${EXPORTER_IMAGE}:${IMAGE_TAG}" ${DBUILD_ARGS} . @rm build/${EXPORTER}/${EXPORTER} .PHONY: all all: check-license deps test exporter # Push images -.PHONY: deploy-images -deploy-images: - @./build/deploy.sh +.PHONY: push +push: + DIMAGE=${IMAGE_ORG}/${EXPORTER_IMAGE} ./build/push.sh .PHONY: check_license check-license: diff --git a/Makefile.buildx.mk b/Makefile.buildx.mk index 05dfec2..b6d636b 100644 --- a/Makefile.buildx.mk +++ b/Makefile.buildx.mk @@ -57,5 +57,5 @@ docker.buildx.exporter: .PHONY: buildx.push.exporter buildx.push.exporter: - BUILDX=true DIMAGE=${IMAGE_ORG}/exporter ./build/buildxpush.sh + BUILDX=true DIMAGE=${IMAGE_ORG}/m-exporter ./build/push.sh diff --git a/build/buildxpush.sh b/build/buildxpush.sh deleted file mode 100755 index 07601f9..0000000 --- a/build/buildxpush.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Copyright 2019-2020 The OpenEBS Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -if [ -z ${DIMAGE} ]; -then - echo "Error: DIMAGE is not specified"; - exit 1 -fi - -function pushBuildx() { - BUILD_TAG="latest" - TARGET_IMG=${DIMAGE} - -# TODO Currently ci builds with commit tag will not be generated, -# since buildx does not support multiple repo - # if not a release build set the tag and ci image - if [ -z "${RELEASE_TAG}" ]; then - return -# BUILD_ID=$(git describe --tags --always) -# BUILD_TAG="${BRANCH}-${BUILD_ID}" -# TARGET_IMG="${DIMAGE}-ci" - fi - - echo "Tagging and pushing ${DIMAGE}:${TAG} as ${TARGET_IMG}:${BUILD_TAG}" - docker buildx imagetools create "${DIMAGE}:${TAG}" -t "${TARGET_IMG}:${BUILD_TAG}" -} - -# if the push is for a buildx build -if [[ ${BUILDX} ]]; then - pushBuildx - exit 0 -fi diff --git a/build/check-diff.sh b/build/check-diff.sh new file mode 100755 index 0000000..9662b35 --- /dev/null +++ b/build/check-diff.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Copyright 2018-2020 The OpenEBS Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script checks if any files are modified by tests like go fmt. + +set -e + +# message to be displayed if test fails. +TEST_NAME=$1 + + +if [[ `git diff --shortstat | wc -l` != 0 ]]; then + echo "Some files got changed after $1";printf "\n";git diff --stat;printf "\n"; exit 1; +fi diff --git a/build/push.sh b/build/push.sh new file mode 100755 index 0000000..7d974a9 --- /dev/null +++ b/build/push.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +# Copyright 2020 The OpenEBS Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -e + +if [ -z ${DIMAGE} ]; +then + echo "Error: DIMAGE is not specified"; + exit 1 +fi + +function pushBuildx() { + BUILD_TAG="latest" + TARGET_IMG=${DIMAGE} + +# TODO Currently ci builds with commit tag will not be generated, +# since buildx does not support multiple repo + # if not a release build set the tag and ci image + if [ -z "${RELEASE_TAG}" ]; then + return +# BUILD_ID=$(git describe --tags --always) +# BUILD_TAG="${BRANCH}-${BUILD_ID}" +# TARGET_IMG="${DIMAGE}-ci" + fi + + echo "Tagging and pushing ${DIMAGE}:${TAG} as ${TARGET_IMG}:${BUILD_TAG}" + docker buildx imagetools create "${DIMAGE}:${TAG}" -t "${TARGET_IMG}:${BUILD_TAG}" +} + +# if the push is for a buildx build +if [[ ${BUILDX} ]]; then + pushBuildx + exit 0 +fi + +# The below steps are required for pushing arch specific images. +# This steps will be removed eventually in favour of buildx-push + +# Generate a unique tag based on the commit and tag +BUILD_ID=$(git describe --tags --always) + +# Determine the current branch +CURRENT_BRANCH="" +if [ -z ${BRANCH} ]; +then + CURRENT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2) +else + CURRENT_BRANCH=${BRANCH} +fi + +#Depending on the branch where builds are generated, +# set the tag CI (fixed) and build tags. +BUILD_TAG="${CURRENT_BRANCH}-${BUILD_ID}" +CI_TAG="${CURRENT_BRANCH}-ci" +if [ ${CURRENT_BRANCH} = "develop" ]; then + CI_TAG="ci" +fi + +echo "Set the fixed ci image tag as: ${CI_TAG}" +echo "Set the build/unique image tag as: ${BUILD_TAG}" + +function TagAndPushImage() { + REPO="$1" + # Trim the `v` from the TAG if it exists + # Example: v1.10.0 maps to 1.10.0 + # Example: 1.10.0 maps to 1.10.0 + # Example: v1.10.0-custom maps to 1.10.0-custom + TAG="${2#v}" + + #Add an option to specify a custom TAG_SUFFIX + #via environment variable. Default is no tag. + #Example suffix could be "-debug" of "-dev" + IMAGE_URI="${REPO}:${TAG}${TAG_SUFFIX}"; + sudo docker tag ${IMAGEID} ${IMAGE_URI}; + echo " push ${IMAGE_URI}"; + sudo docker push ${IMAGE_URI}; +} + + +if [ ! -z "${DNAME}" ] && [ ! -z "${DPASS}" ]; +then + sudo docker login -u "${DNAME}" -p "${DPASS}"; + + # Push CI tagged image - :ci or :branch-ci + TagAndPushImage "${DIMAGE}" "${CI_TAG}" + + # Push unique tagged image - :master- or :branch- + # This unique/build image will be pushed to corresponding ci repo. + TagAndPushImage "${DIMAGE}-ci" "${BUILD_TAG}" + + if [ ! -z "${RELEASE_TAG}" ] ; + then + # Push with different tags if tagged as a release + # When github is tagged with a release, then Travis will + # set the release tag in env RELEASE_TAG + TagAndPushImage "${DIMAGE}" "${RELEASE_TAG}" + TagAndPushImage "${DIMAGE}" "latest" + fi; +else + echo "No docker credentials provided. Skip uploading ${DIMAGE} to docker hub"; +fi; + +# Push ci image to quay.io for security scanning +if [ ! -z "${QNAME}" ] && [ ! -z "${QPASS}" ]; +then + sudo docker login -u "${QNAME}" -p "${QPASS}" quay.io; + + # Push CI tagged image - :ci or :branch-ci + TagAndPushImage "quay.io/${DIMAGE}" "${CI_TAG}" + + if [ ! -z "${RELEASE_TAG}" ] ; + then + # Push with different tags if tagged as a release + # When github is tagged with a release, then Travis will + # set the release tag in env RELEASE_TAG + # Trim the `v` from the RELEASE_TAG if it exists + TagAndPushImage "quay.io/${DIMAGE}" "${RELEASE_TAG}" + TagAndPushImage "quay.io/${DIMAGE}" "latest" + fi; +else + echo "No docker credentials provided. Skip uploading ${DIMAGE} to quay"; +fi; + diff --git a/build/test.sh b/build/test.sh new file mode 100755 index 0000000..a3a1567 --- /dev/null +++ b/build/test.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Copyright 2018-2020 The OpenEBS Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script runs tests and generates a report file. + +set -e + +# architecute on which tests need to be run +ARCH=$1 + +if [ -z "$ARCH" ]; then + echo "platform not specified for running tests. Exiting." + exit 1 +fi + +# currently tests are run only for amd64 +#if [ "$ARCH" != "amd64" ]; then +# exit 0 +#fi + +echo "" > coverage.txt +PACKAGES=$(go list ./... | grep -v '/vendor/\|/pkg/apis/\|/pkg/client/\|integration_test') +for d in $PACKAGES; do + go test -coverprofile=profile.out -covermode=atomic "$d" + if [ -f profile.out ]; then + cat profile.out >> coverage.txt + rm profile.out + fi +done