TT-10886 added initial proposal for tests in CI

.github/workflows/ci-tests.yml

@@ -0,0 +1,140 @@
+name: Go Test Workflow
+
+on:
+  push:
+    branches:
+      - master
+      - release-**
+  pull_request:
+    branches:
+      - master
+      - release-**
+env:
+  TYK_IB_STORAGE_STORAGETYPE: file
+
+permissions:
+  contents: read
+
+jobs:
+  golangci-lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Tyk Identity Broker
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - name: Fetch base branch
+        if: ${{ github.event_name == 'pull_request' }}
+        run: git fetch origin ${{ github.base_ref }}
+      - name: golangci-lint
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: latest
+          args: --out-format=checkstyle:golanglint.xml --timeout=600s --max-issues-per-linter=0 --max-same-issues=0 --new-from-rev=origin/${{ github.base_ref }}
+      - uses: actions/upload-artifact@v2
+        with:
+          name: golangcilint
+          retention-days: 1
+          path: |
+            golanglint.xml
+
+  ci-test:
+    name: '${{ matrix.databases }}'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        databases:
+          - mongo-mgo
+          - mongo-official
+          - file
+        redis-version: [5]
+        mongodb-version: [4.2]
+    steps:
+     - name: Checkout Code
+       uses: actions/checkout@v4
+       with:
+        fetch-depth: 2
+
+     - name: Setup Go
+       uses: actions/setup-go@v2
+       with:
+        go-version: '^1.21'
+
+     - name: Install Dependencies and basic hygiene test
+       id: hygiene
+       run: |
+        go install golang.org/x/tools/cmd/goimports@latest
+
+     - name: Start Redis
+       uses: supercharge/redis-github-action@1.2.0
+       with:
+          redis-version: ${{ matrix.redis-version }}
+
+     - name: Start MongoDB
+       uses: supercharge/mongodb-github-action@1.2.0
+       with:
+          mongodb-version: '${{ matrix.mongodb-version }}'
+
+     - name: Run tests
+       run: |
+        ./bin/ci-tests.sh ${{ matrix.databases }}
+     - uses: actions/upload-artifact@v2
+       with:
+         name: coverage
+         retention-days: 1
+         path: |
+           *cov
+
+  sonar-cloud-analysis:
+    runs-on: ubuntu-latest
+    needs: [ci-test, golangci-lint]
+    steps:
+      - name: Checkout TIB
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - name: Fetch base branch
+        if: ${{ github.event_name == 'pull_request' }}
+        run: git fetch origin ${{ github.base_ref }}
+      - name: Setup Golang
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.21
+      - name: Download coverage artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: coverage
+      - name: Download golangcilint artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: golangcilint
+      - name: Check reports existence
+        id: check_files
+        uses: andstor/file-existence-action@v1
+        with:
+          files: '*.cov, golanglint.xml'
+      - name: Install Dependencies
+        run: >
+          go install github.com/wadey/gocovmerge@latest
+
+      - name: merge reports
+        run: |
+          ./bin/merge-cov.sh
+      - name: SonarCloud Scan
+        uses: sonarsource/sonarcloud-github-action@master
+        with:
+         args: >
+          -Dsonar.organization=tyktechnologies
+          -Dsonar.projectKey=TykTechnologies_tyk-identity-broker
+          -Dsonar.sources=.
+          -Dsonar.exclusions=ci/**
+          -Dsonar.coverage.exclusions=**/*_test.go,**/mocks/*.go
+          -Dsonar.test.inclusions=**/*_test.go
+          -Dsonar.tests=.
+          -Dsonar.go.coverage.reportPaths=*.cov
+          -Dsonar.go.golangci-lint.reportPaths=golanglint.xml
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

bin/ci-tests.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+set -e
+
+export GO111MODULE=on
+
+# print a command and execute it
+show() {
+ echo "$@" >&2
+ eval "$@"
+}
+
+fatal() {
+ echo "$@" >&2
+ exit 1
+}
+
+TEST_TIMEOUT=10m
+db="file"
+
+if [[ -n $1 ]]; then
+    db=$1
+fi
+
+if [[ $db = "mongo-mgo" ]]; then
+    export TYK_IB_STORAGE_STORAGETYPE="mongo"
+    export TYK_IB_STORAGE_MONGOCONF_MONGOURL="mongodb://localhost/tyk_identity_broker"
+    export TYK_IB_STORAGE_MONGOCONF_DRIVER="mgo"
+    : # do nothing
+elif [[ $db = "mongo-official" ]]; then
+    export TYK_IB_STORAGE_STORAGETYPE="mongo"
+    export TYK_IB_STORAGE_MONGOCONF_MONGOURL="mongodb://localhost/tyk_identity_broker"
+    export TYK_IB_STORAGE_MONGOCONF_DRIVER="mongo-go"
+elif [[ $db = "file" ]]; then
+    export TYK_IB_STORAGE_STORAGETYPE="file"
+    export TYK_IB_PROFILEDIR="./"
+else
+    fatal "unsupported database: $db"
+fi
+
+echo "Running with $TYK_IB_STORAGE_STORAGETYPE database using $TYK_IB_STORAGE_MONGOCONF_DRIVER driver"
+
+show go vet . || fatal "go vet errored"
+
+GO_FILES=$(find * -name '*.go' )
+
+echo "Formatting checks..."
+
+FMT_FILES="$(gofmt -s -l ${GO_FILES})"
+if [[ -n ${FMT_FILES} ]]; then
+	fatal "Run 'gofmt -s -w' on these files:\n$FMT_FILES"
+fi
+
+echo "gofmt check is ok!"
+
+IMP_FILES="$(goimports -l ${GO_FILES})"
+if [[ -n ${IMP_FILES} ]]; then
+	fatal "Run 'goimports -w' on these files:\n$IMP_FILES"
+fi
+
+echo "goimports check is ok!"
+
+for pkg in $(go list github.com/TykTechnologies/tyk-identity-broker/...);
+do
+    race="-race"
+    echo "Testing... $pkg"
+    coveragefile=`echo "$pkg-$db" | awk -F/ '{print $NF}'`
+    show go test -timeout ${TEST_TIMEOUT} ${race} --coverprofile=${coveragefile}.cov -v ${pkg}
+done

bin/merge-cov.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+
+export GO111MODULE=on
+
+# print a command and execute it
+show() {
+ echo "$@" >&2
+ eval "$@"
+}
+
+fatal() {
+ echo "$@" >&2
+ exit 1
+}
+
+for pkg in $(go list github.com/TykTechnologies/tyk-identity-broker/...);
+do
+    coveragefile=`echo "$pkg.cov" | awk -F/ '{print $NF}'`
+    mgo_cov=`echo "$pkg-mongo-mgo.cov" | awk -F/ '{print $NF}'`
+    mongo_cov=`echo "$pkg-mongo-official.cov" | awk -F/ '{print $NF}'`
+    file_cov=`echo "$pkg-file.cov" | awk -F/ '{print $NF}'`
+    show gocovmerge $mongo_cov $mgo_cov $file_cov > $coveragefile
+    rm $mongo_cov $mgo_cov $file_cov
+done

data_loader/data_loader_test.go

@@ -1,31 +1,79 @@
-//go:build test_mongo
-// +build test_mongo
-
-package data_loader_test
+package data_loader
 
 import (
+	"os"
 	"testing"
 
+	"github.com/TykTechnologies/storage/persistent"
+	"github.com/TykTechnologies/tyk-identity-broker/backends"
+	"github.com/stretchr/testify/assert"
+
 	"github.com/TykTechnologies/tyk-identity-broker/configuration"
-	"github.com/TykTechnologies/tyk-identity-broker/data_loader"
 )
 
 func TestCreateDataMongoLoader(t *testing.T) {
+	isMongo := isMongoEnv()
+	if !isMongo {
+		t.Skip()
+	}
+
+	url, driver := MongoEnvConf()
+
 	conf := configuration.Configuration{
 		Storage: &configuration.Storage{
-			StorageType: configuration.MONGO,
+			StorageType: "mongo",
 			MongoConf: &configuration.MongoConf{
-				MongoURL: "mongodb://tyk-mongo:27017/tyk_tib",
+				MongoURL: url,
+				Driver:   driver,
 			},
 		},
 	}
 
-	dataLoader, err := data_loader.CreateDataLoader(conf, nil)
+	dataLoader, err := CreateDataLoader(conf, "")
 	if err != nil {
 		t.Fatalf("creating Mongo data loader: %v", err)
 	}
 
-	if _, ok := dataLoader.(*data_loader.MongoLoader); !ok {
+	if _, ok := dataLoader.(*MongoLoader); !ok {
 		t.Fatalf("type of data loader is not correct; expected '*data_loader.MongoLoader' but got '%T'", dataLoader)
 	}
 }
+
+func TestFlush(t *testing.T) {
+
+	isMongo := isMongoEnv()
+	if !isMongo {
+		t.Skip()
+	}
+
+	url, driver := MongoEnvConf()
+	loader := MongoLoader{}
+
+	var err error
+	loader.store, err = persistent.NewPersistentStorage(&persistent.ClientOpts{
+		ConnectionString: url,
+		UseSSL:           false,
+		Type:             driver,
+	})
+
+	assert.Nil(t, err)
+
+	authStore := &backends.InMemoryBackend{}
+	err = loader.Flush(authStore)
+	assert.Nil(t, err)
+}
+
+func isMongoEnv() bool {
+	storageType := os.Getenv("TYK_IB_STORAGE_STORAGETYPE")
+
+	// Check if MongoDB is the storage type and both URL and driver are set
+	return storageType == "mongo"
+}
+
+func MongoEnvConf() (string, string) {
+	mongoURL := os.Getenv("TYK_IB_STORAGE_MONGOCONF_MONGOURL")
+	mongoDriver := os.Getenv("TYK_IB_STORAGE_MONGOCONF_DRIVER")
+
+	return mongoURL, mongoDriver
+
+}

data_loader/mongo_loader.go

@@ -69,6 +69,16 @@ func (m *MongoLoader) LoadIntoStore(store tap.AuthRegisterBackend) error {
 	return nil
 }
 
+func handleEmptyProfilesError(err error) error {
+	if err != nil {
+		if !utils.IsErrNoRows(err) {
+			dataLogger.WithError(err).Error("emptying profiles collection")
+			return err
+		}
+	}
+	return nil
+}
+
 // Flush creates a backup of the current loaded config
 func (m *MongoLoader) Flush(store tap.AuthRegisterBackend) error {
 	//read all
@@ -77,11 +87,8 @@ func (m *MongoLoader) Flush(store tap.AuthRegisterBackend) error {
 
 	//empty to store new changes
 	err := m.store.Delete(context.Background(), tap.Profile{}, nil)
-	if err != nil {
-		if !utils.IsErrNoRows(err) {
-			dataLogger.WithError(err).Error("emptying profiles collection")
-			return err
-		}
+	if handledErr := handleEmptyProfilesError(err); handledErr != nil {
+		return handledErr
 	}
 
 	for _, p := range updatedSet {