Export migration stats

pkg/utils/export_stats.go

@@ -0,0 +1,238 @@
+package utils
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	storkv1 "github.com/libopenstorage/stork/pkg/apis/stork/v1alpha1"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	AetosStatsURL = "https://aetos.pwx.purestorage.com/dashboard/stats?stats_type=MigrationDemoFinal&limit=100"
+)
+
+var StorkVersion string
+var PortworxVersion string
+
+type StatsExportType struct {
+	//	id        OidType            `json: "_id",omitempty`
+	Name      string             `json:"name",omitempty`
+	Product   string             `json:"product",omitempty`
+	Version   string             `json:"version",omitempty`
+	StatsType string             `json:"statsType",omitempty`
+	Data      MigrationStatsType `json:"data",omitempty`
+}
+
+type MigrationStatsType struct {
+	CreatedOn                       string      `json:"created",omitempty`
+	TotalNumberOfVolumes            json.Number `json:"totalNumberOfVolumes",omitempty`
+	NumOfMigratedVolumes            json.Number `json:"numOfMigratedVolumes",omitempty`
+	TotalNumberOfResources          json.Number `json:"totalNumberOfResources",omitempty`
+	NumOfMigratedResources          json.Number `json:"numOfMigratedResources",omitempty`
+	TotalBytesMigrated              json.Number `json:"totalBytesMigrated",omitempty`
+	ElapsedTimeForVolumeMigration   string      `json:"elapsedTimeForVolumeMigration",omitempty`
+	ElapsedTimeForResourceMigration string      `json:"elapsedTimeForResourceMigration",omitempty`
+	Application                     string      `json:"application",omitempty`
+	StorkVersion                    string      `json:"storkVersion",omitempty`
+	PortworxVersion                 string      `json:"portworxVersion",omitempty`
+}
+
+//type OidType struct {
+//	oid string `json: "$oid"`
+//}
+
+type AllStats []StatsExportType
+
+func GetMigrationStatsFromAetos(url string) ([]StatsExportType, error) {
+	client := http.Client{Timeout: time.Second * 3}
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("error querying Aetos for stats: %v", err)
+	}
+
+	req.Header.Add("Content-Type", "application/json")
+
+	q := req.URL.Query()
+	q.Add("format", "json")
+	req.URL.RawQuery = q.Encode()
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error querying Aetos metadata: %v", err)
+	}
+
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("error querying Aetos metadata: Code %d returned for url %s", resp.StatusCode, req.URL)
+	}
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("error querying Aetos metadata: %v", err)
+	}
+	if len(body) == 0 {
+		return nil, fmt.Errorf("error querying Aetos metadata: Empty response")
+	}
+
+	//fmt.Printf("\nBody of response: %v\n\n", string(body))
+	data := AllStats{}
+	err = json.Unmarshal(body, &data)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing Aetos metadata: %v", err)
+	}
+
+	defer func() {
+		err := resp.Body.Close()
+		if err != nil {
+			logrus.Errorf("Error closing body when getting Aetos data: %v", err)
+		}
+	}()
+	return data, nil
+}
+
+func WriteMigrationStatsToAetos(data StatsExportType) error {
+	body, err := json.Marshal(data)
+	if err != nil {
+		return fmt.Errorf("failed to marshal: %v", err)
+	}
+	logrus.Infof("RK=> Marshalled data\n%s\n", body)
+
+	transCfg := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{Transport: transCfg}
+
+	resp, err := client.Post("https://aetos.pwx.purestorage.com/dashboard/stats", "application/json", bytes.NewBuffer(body))
+	if err != nil {
+		return fmt.Errorf("post request to Aetos failed: %v", err)
+	}
+	defer resp.Body.Close()
+	respBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		//Failed to read response.
+		return fmt.Errorf("response from Aetos failed: %v", err)
+	}
+	jsonStr := string(respBody)
+
+	if resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusOK {
+		logrus.Infof("Stats successfully pushed to DB. Response: %s", jsonStr)
+	} else {
+		return fmt.Errorf("post failed with Reponse status: %s, %s", resp.Status, jsonStr)
+	}
+
+	return nil
+}
+
+func MockStat() StatsExportType {
+	mockStat := StatsExportType{
+		Name:      "stork_integration_test",
+		Product:   "stork",
+		StatsType: "migration_stats_mock",
+		Version:   "v1alpha1",
+		Data: MigrationStatsType{
+			TotalNumberOfVolumes:            "1",
+			NumOfMigratedVolumes:            "1",
+			TotalNumberOfResources:          "5",
+			NumOfMigratedResources:          "1",
+			TotalBytesMigrated:              "12345",
+			ElapsedTimeForVolumeMigration:   "11.1111s",
+			ElapsedTimeForResourceMigration: "12321.3453s",
+			Application:                     "mock_app",
+			StorkVersion:                    StorkVersion,
+			PortworxVersion:                 PortworxVersion,
+		},
+	}
+	return mockStat
+}
+
+func NewStat() StatsExportType {
+	newStat := StatsExportType{
+		Name:      "stork_integration_test",
+		Product:   "stork",
+		StatsType: "MigrationDemoFinal",
+		Version:   "v1alpha1",
+		Data: MigrationStatsType{
+			TotalNumberOfVolumes:            "",
+			Application:                     "",
+			NumOfMigratedVolumes:            "",
+			TotalNumberOfResources:          "",
+			NumOfMigratedResources:          "",
+			TotalBytesMigrated:              "",
+			ElapsedTimeForVolumeMigration:   "",
+			ElapsedTimeForResourceMigration: "",
+			StorkVersion:                    StorkVersion,
+			PortworxVersion:                 PortworxVersion,
+		},
+	}
+	return newStat
+}
+
+func GetExportableStatsFromMigrationObject(mig *storkv1.Migration, statArgs ...string) StatsExportType {
+	exportStats := NewStat()
+
+	for i := 0; i < len(statArgs); i++ {
+		switch i {
+		case 0:
+			exportStats.Name = statArgs[0]
+		case 1:
+			exportStats.Product = statArgs[1]
+		case 2:
+			exportStats.StatsType = statArgs[2]
+		case 3:
+			exportStats.Version = statArgs[3]
+		}
+	}
+	exportStats.Data.Application = getResourceNamesFromMigration(mig)
+
+	exportStats.Data.CreatedOn = (mig.GetCreationTimestamp()).Format("2006-01-02 15:04:05")
+
+	exportStats.Data.TotalNumberOfVolumes = json.Number(strconv.FormatUint(mig.Status.Summary.TotalNumberOfVolumes, 10))
+
+	exportStats.Data.NumOfMigratedVolumes = json.Number(strconv.FormatUint(mig.Status.Summary.NumberOfMigratedVolumes, 10))
+
+	exportStats.Data.TotalNumberOfResources = json.Number(strconv.FormatUint(mig.Status.Summary.TotalNumberOfResources, 10))
+
+	exportStats.Data.NumOfMigratedResources = json.Number(strconv.FormatUint(mig.Status.Summary.NumberOfMigratedResources, 10))
+
+	exportStats.Data.TotalBytesMigrated = json.Number(strconv.FormatUint(mig.Status.Summary.TotalBytesMigrated, 10))
+
+	exportStats.Data.ElapsedTimeForVolumeMigration = mig.Status.Summary.ElapsedTimeForVolumeMigration
+
+	exportStats.Data.ElapsedTimeForResourceMigration = mig.Status.Summary.ElapsedTimeForResourceMigration
+
+	PrettyStruct(exportStats)
+
+	return exportStats
+
+}
+
+func PrettyStruct(data interface{}) {
+	val, err := json.MarshalIndent(data, "", "    ")
+	if err != nil {
+		logrus.Panicf("Failed to prettify data")
+	}
+	fmt.Printf("Exportable migration data: %v", string(val))
+}
+
+func getResourceNamesFromMigration(mig *storkv1.Migration) string {
+	var resourceList []string
+	for _, resource := range mig.Status.Resources {
+		if resource.Kind == "Deployment" || resource.Kind == "StatefulSet" {
+			resourceList = append(resourceList, resource.Name)
+		}
+	}
+	if len(resourceList) > 1 {
+		// return comma separated list of apps if there are multiple apps
+		return strings.Join(resourceList, ",")
+	} else if len(resourceList) == 1 {
+		return resourceList[0]
+	}
+
+	logrus.Info("App name not found for pushing to DB.")
+	return ""
+}

test/integration_test/common_test.go

@@ -26,6 +26,7 @@ import (
 	storkv1 "github.com/libopenstorage/stork/pkg/apis/stork/v1alpha1"
 	"github.com/libopenstorage/stork/pkg/schedule"
 	"github.com/libopenstorage/stork/pkg/storkctl"
+	"github.com/libopenstorage/stork/pkg/utils"
 	"github.com/libopenstorage/stork/pkg/version"
 	"github.com/portworx/sched-ops/k8s/apps"
 	"github.com/portworx/sched-ops/k8s/batch"
@@ -132,6 +133,9 @@ const (
 	testrailUserNameVar        = "TESTRAIL_USERNAME"
 	testrailPasswordVar        = "TESTRAIL_PASSWORD"
 	testrailMilestoneVar       = "TESTRAIL_MILESTONE"
+
+	statsExportName    = "stork_integration_test"
+	statsExportProduct = "stork"
 )
 
 var nodeDriver node.Driver
@@ -152,16 +156,19 @@ var backupLocationPath string
 var genericCsiConfigMap string
 var externalTest bool
 var storkVersionCheck bool
+var storkVersion string
+var pxVersion string
 var cloudDeletionValidate bool
 var isInternalLBAws bool
 var pxNamespace string
-var storkVersion string
 var testrailHostname string
 var testrailUsername string
 var testrailPassword string
 var testrailSetupSuccessful bool
+var exportStats bool
 var bidirectionalClusterpair bool
 
+
 func TestSnapshot(t *testing.T) {
 	t.Run("testSnapshot", testSnapshot)
 	t.Run("testSnapshotRestore", testSnapshotRestore)
@@ -276,12 +283,20 @@ func setup() error {
 			return fmt.Errorf("stork version not found in configmap: %s", cmName)
 		}
 		storkVersion = getStorkVersion(ver)
+		utils.StorkVersion = ver
+
 		if storkVersionCheck == true {
 			if getStorkVersion(ver) != getStorkVersion(version.Version) {
 				return fmt.Errorf("stork version mismatch, found: %s, expected: %s", getStorkVersion(ver), getStorkVersion(version.Version))
 			}
 		}
 	}
+	stc, err := operator.Instance().ListStorageClusters("kube-system")
+	if err != nil {
+		logrus.Warnf("failed to list PX storage cluster during setup: %v, probably a daemonset install for portworx", err)
+	} else {
+		utils.PortworxVersion = oputils.GetPortworxVersion(&stc.Items[0]).Original()
+	}
 
 	isInternalLBAws, err = strconv.ParseBool(os.Getenv(internalLBAws))
 	if err == nil {
@@ -1498,6 +1513,10 @@ func TestMain(m *testing.M) {
 		"stork-version-check",
 		false,
 		"Turn on/off stork version check before running tests. Default off.")
+	flag.BoolVar(&exportStats,
+		"export-stats",
+		true,
+		"Turn on/off exporting stats to aetos DB. Default on.")
 	flag.BoolVar(&bidirectionalClusterpair,
 		"bidirectional-cluster-pair",
 		false,
@@ -1539,3 +1558,28 @@ func updateClusterDomain(t *testing.T, clusterDomains *storkv1.ClusterDomains, a
 		}
 	}
 }
+
+func ExportMigrationStats(ctx *scheduler.Context, name string) error {
+	if !exportStats {
+		logrus.Infof("Export flag is off, won't export migration stats")
+		return nil
+	}
+	for _, spec := range ctx.App.SpecList {
+		if obj, ok := spec.(*storkv1.Migration); ok {
+			logrus.Infof("Found migration object: %s", obj.Name)
+			mig, err := storkops.Instance().GetMigration(obj.Name, obj.Namespace)
+			if err != nil {
+				return fmt.Errorf("Failed to get migration %s in namespace: %s: %v", obj.Name, obj.Namespace, err)
+			}
+			//stats := utils.GetExportableStatsFromMigrationObject(mig, statsExportName, statsExportProduct, name, utils.StorkVersion)
+			stats := utils.GetExportableStatsFromMigrationObject(mig)
+			err = utils.WriteMigrationStatsToAetos(stats)
+			if err != nil {
+				return fmt.Errorf("Failed to write stats: %v", err)
+			}
+		} else {
+			logrus.Info("Not a migration object")
+		}
+	}
+	return nil
+}

test/integration_test/export_stats_test.go

@@ -0,0 +1,46 @@
+//go:build integrationtest
+// +build integrationtest
+
+package integrationtest
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/libopenstorage/stork/pkg/utils"
+)
+
+const (
+	aetosStatsURL = "http://aetos.pwx.purestorage.com/dashboard/stats?stats_type=migration_stats_new&limit=100"
+)
+
+func TestExportStatsGetStats(t *testing.T) {
+	fmt.Println("Hello")
+	data, err := utils.GetMigrationStatsFromAetos(aetosStatsURL)
+	require.NoError(t, err, "Failed to get stats: %v")
+
+	prettyData, err := PrettyStruct(data)
+	require.NoError(t, err, "Failed to print pretty data: %v")
+	fmt.Println(prettyData)
+}
+
+func TestExportStatsPushMockStats(t *testing.T) {
+	err := utils.WriteMigrationStatsToAetos(NewStat())
+	require.NoError(t, err, "Failed to write stats: %v")
+}
+
+func NewStat() utils.StatsExportType {
+	mockStat := utils.StatsExportType{}
+	return mockStat
+}
+
+func PrettyStruct(data interface{}) (string, error) {
+	val, err := json.MarshalIndent(data, "", "    ")
+	if err != nil {
+		return "", err
+	}
+	return string(val), nil
+}