API for exporting stats

Signed-off-by: Rohit-PX <rkulkarni@purestorage.com>

pkg/utils/export_stats.go

@@ -0,0 +1,218 @@
+package utils
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	storkv1 "github.com/libopenstorage/stork/pkg/apis/stork/v1alpha1"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	AetosStatsURL = "http://aetos.pwx.purestorage.com/dashboard/stats?stats_type=MigrationDemoFinal&limit=100"
+)
+
+var StorkVersion string
+var PortworxVersion string
+
+type StatsExportType struct {
+	id        OidType            `json: "_id",omitempty`
+	Name      string             `json: "name",omitempty`
+	Product   string             `json: "product",omitempty`
+	Version   string             `json: "version",omitempty`
+	StatsType string             `json: "statsType",omitempty`
+	Data      MigrationStatsType `json: "data",omitempty`
+}
+
+type MigrationStatsType struct {
+	CreatedOn                       string      `json: "created",omitempty`
+	TotalNumberOfVolumes            json.Number `json: "totalNumberOfVolumes",omitempty`
+	NumOfMigratedVolumes            json.Number `json: "numOfMigratedVolumes",omitempty`
+	TotalNumberOfResources          json.Number `json: "totalNumberOfResources",omitempty`
+	NumOfMigratedResources          json.Number `json: "numOfMigratedResources",omitempty`
+	TotalBytesMigrated              json.Number `json: "totalBytesMigrated",omitempty`
+	ElapsedTimeForVolumeMigration   string      `json: "elapsedTimeForVolumeMigration",omitempty`
+	ElapsedTimeForResourceMigration string      `json: "elapsedTimeForResourceMigration",omitempty`
+	Application                     string      `json: "application",omitempty`
+	StorkVersion                    string      `json: "storkVersion",omitempty`
+	PortworxVersion                 string      `json: "portworxVersion",omitempty`
+}
+
+type OidType struct {
+	oid string `json: "$oid"`
+}
+
+type AllStats []StatsExportType
+
+func GetMigrationStatsFromAetos(url string) ([]StatsExportType, error) {
+	client := http.Client{Timeout: time.Second * 3}
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("error querying Aetos for stats: %v", err)
+	}
+
+	req.Header.Add("Content-Type", "application/json")
+
+	q := req.URL.Query()
+	q.Add("format", "json")
+	req.URL.RawQuery = q.Encode()
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error querying Aetos metadata: %v", err)
+	}
+
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("error querying Aetos metadata: Code %d returned for url %s", resp.StatusCode, req.URL)
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("error querying Aetos metadata: %v", err)
+	}
+	if len(body) == 0 {
+		return nil, fmt.Errorf("error querying Aetos metadata: Empty response")
+	}
+
+	//fmt.Printf("\nBody of response: %v\n\n", string(body))
+	data := AllStats{}
+	err = json.Unmarshal(body, &data)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing Aetos metadata: %v", err)
+	}
+
+	defer func() {
+		err := resp.Body.Close()
+		if err != nil {
+			logrus.Errorf("Error closing body when getting Aetos data: %v", err)
+		}
+	}()
+	return data, nil
+}
+
+func WriteMigrationStatsToAetos(data StatsExportType) error {
+	body, err := json.Marshal(data)
+	if err != nil {
+		return fmt.Errorf("failed to marshal: %v", err)
+	}
+	resp, err := http.Post("http://aetos.pwx.purestorage.com/dashboard/stats", "application/json", bytes.NewBuffer(body))
+	if err != nil {
+		return fmt.Errorf("post request to Aetos failed: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusOK {
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			//Failed to read response.
+			return fmt.Errorf("response from Aetos failed: %v", err)
+		}
+
+		jsonStr := string(body)
+		logrus.Infof("Stats successfully pushed to DB. Response: ", jsonStr)
+	} else {
+		return fmt.Errorf("Get failed with Reponse status: %s", resp.Status)
+	}
+
+	return nil
+}
+
+func MockStat() StatsExportType {
+	mockStat := StatsExportType{
+		Name:      "stork_integration_test",
+		Product:   "stork",
+		StatsType: "migration_stats_mock",
+		Version:   "v1alpha1",
+		Data: MigrationStatsType{
+			TotalNumberOfVolumes:            "1",
+			NumOfMigratedVolumes:            "1",
+			TotalNumberOfResources:          "5",
+			NumOfMigratedResources:          "1",
+			TotalBytesMigrated:              "12345",
+			ElapsedTimeForVolumeMigration:   "11.1111s",
+			ElapsedTimeForResourceMigration: "12321.3453s",
+			Application:                     "mock_app",
+			StorkVersion:                    StorkVersion,
+			PortworxVersion:                 PortworxVersion,
+		},
+	}
+	return mockStat
+}
+
+func NewStat() StatsExportType {
+	newStat := StatsExportType{
+		Name:      "stork_integration_test",
+		Product:   "stork",
+		StatsType: "MigrationDemoFinal",
+		Version:   "v1alpha1",
+		Data: MigrationStatsType{
+			TotalNumberOfVolumes:            "",
+			Application:                     "",
+			NumOfMigratedVolumes:            "",
+			TotalNumberOfResources:          "",
+			NumOfMigratedResources:          "",
+			TotalBytesMigrated:              "",
+			ElapsedTimeForVolumeMigration:   "",
+			ElapsedTimeForResourceMigration: "",
+			StorkVersion:                    StorkVersion,
+			PortworxVersion:                 PortworxVersion,
+		},
+	}
+	return newStat
+}
+
+func GetExportableStatsFromMigrationObject(mig *storkv1.Migration) StatsExportType {
+	exportStats := NewStat()
+
+	exportStats.Data.Application = getResourceNamesFromMigration(mig)
+
+	exportStats.Data.CreatedOn = (mig.GetCreationTimestamp()).Format("2006-01-02 15:04:05")
+
+	exportStats.Data.TotalNumberOfVolumes = json.Number(strconv.FormatUint(mig.Status.Summary.TotalNumberOfVolumes, 10))
+
+	exportStats.Data.NumOfMigratedVolumes = json.Number(strconv.FormatUint(mig.Status.Summary.NumberOfMigratedVolumes, 10))
+
+	exportStats.Data.TotalNumberOfResources = json.Number(strconv.FormatUint(mig.Status.Summary.TotalNumberOfResources, 10))
+
+	exportStats.Data.NumOfMigratedResources = json.Number(strconv.FormatUint(mig.Status.Summary.NumberOfMigratedResources, 10))
+
+	exportStats.Data.TotalBytesMigrated = json.Number(strconv.FormatUint(mig.Status.Summary.TotalBytesMigrated, 10))
+
+	exportStats.Data.ElapsedTimeForVolumeMigration = mig.Status.Summary.ElapsedTimeForVolumeMigration
+
+	exportStats.Data.ElapsedTimeForResourceMigration = mig.Status.Summary.ElapsedTimeForResourceMigration
+
+	PrettyStruct(exportStats)
+
+	return exportStats
+
+}
+
+func PrettyStruct(data interface{}) {
+	val, err := json.MarshalIndent(data, "", "    ")
+	if err != nil {
+		logrus.Panicf("Failed to prettify data")
+	}
+	fmt.Printf("Exportable migration data: %v", string(val))
+}
+
+func getResourceNamesFromMigration(mig *storkv1.Migration) string {
+	var resourceList []string
+	for _, resource := range mig.Status.Resources {
+		if resource.Kind == "Deployment" || resource.Kind == "StatefulSet" {
+			resourceList = append(resourceList, resource.Name)
+		}
+	}
+	if len(resourceList) > 1 {
+		// return comma separated list of apps if there are multiple apps
+		return strings.Join(resourceList, ",")
+	} else if len(resourceList) == 1 {
+		return resourceList[0]
+	}
+
+	logrus.Info("App name not found for pushing to DB.")
+	return ""
+}

test/integration_test/common_test.go

@@ -26,6 +26,7 @@ import (
 	storkv1 "github.com/libopenstorage/stork/pkg/apis/stork/v1alpha1"
 	"github.com/libopenstorage/stork/pkg/schedule"
 	"github.com/libopenstorage/stork/pkg/storkctl"
+	"github.com/libopenstorage/stork/pkg/utils"
 	"github.com/libopenstorage/stork/pkg/version"
 	"github.com/portworx/sched-ops/k8s/apps"
 	"github.com/portworx/sched-ops/k8s/batch"
@@ -152,10 +153,11 @@ var backupLocationPath string
 var genericCsiConfigMap string
 var externalTest bool
 var storkVersionCheck bool
+var storkVersion string
+var pxVersion string
 var cloudDeletionValidate bool
 var isInternalLBAws bool
 var pxNamespace string
-var storkVersion string
 var testrailHostname string
 var testrailUsername string
 var testrailPassword string
@@ -275,12 +277,20 @@ func setup() error {
 			return fmt.Errorf("stork version not found in configmap: %s", cmName)
 		}
 		storkVersion = getStorkVersion(ver)
+		utils.StorkVersion = ver
+
 		if storkVersionCheck == true {
 			if getStorkVersion(ver) != getStorkVersion(version.Version) {
 				return fmt.Errorf("stork version mismatch, found: %s, expected: %s", getStorkVersion(ver), getStorkVersion(version.Version))
 			}
 		}
 	}
+	stc, err := operator.Instance().ListStorageClusters("kube-system")
+	if err != nil {
+		logrus.Warnf("failed to list PX storage cluster during setup: %v, probably a daemonset install for portworx", err)
+	} else {
+		utils.PortworxVersion = oputils.GetPortworxVersion(&stc.Items[0]).Original()
+	}
 
 	isInternalLBAws, err = strconv.ParseBool(os.Getenv(internalLBAws))
 	if err == nil {

test/integration_test/export_stats_test.go

@@ -0,0 +1,60 @@
+//go:build integrationtest
+// +build integrationtest
+
+package integrationtest
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/libopenstorage/stork/pkg/utils"
+)
+
+const (
+	aetosStatsURL = "http://aetos.pwx.purestorage.com/dashboard/stats?stats_type=migration_stats_new&limit=100"
+)
+
+func TestExportStatsGetStats(t *testing.T) {
+	fmt.Println("Hello")
+	data, err := utils.GetMigrationStatsFromAetos(aetosStatsURL)
+	require.NoError(t, err, "Failed to get stats: %v")
+
+	prettyData, err := PrettyStruct(data)
+	require.NoError(t, err, "Failed to print pretty data: %v")
+	fmt.Println(prettyData)
+}
+
+func TestExportStatsPushMockStats(t *testing.T) {
+	err := utils.WriteMigrationStatsToAetos(NewStat())
+	require.NoError(t, err, "Failed to write stats: %v")
+}
+
+func NewStat() utils.StatsExportType {
+	mockStat := utils.StatsExportType{
+		Name:      "stork_integration_test",
+		Product:   "stork",
+		StatsType: "migration_stats_mock",
+		Version:   "v1alpha1",
+		Data: utils.MigrationStatsType{
+			TotalNumberOfVolumes:            "1",
+			NumOfMigratedVolumes:            "1",
+			TotalNumberOfResources:          "5",
+			NumOfMigratedResources:          "1",
+			TotalBytesMigrated:              "12345",
+			ElapsedTimeForVolumeMigration:   "11.1111s",
+			ElapsedTimeForResourceMigration: "12321.3453s",
+		},
+	}
+	return mockStat
+}
+
+func PrettyStruct(data interface{}) (string, error) {
+	val, err := json.MarshalIndent(data, "", "    ")
+	if err != nil {
+		return "", err
+	}
+	return string(val), nil
+}