diff --git a/go.mod b/go.mod index 3db4e1b3a8..0fe10d00a6 100644 --- a/go.mod +++ b/go.mod @@ -70,6 +70,7 @@ replace ( require ( github.com/Microsoft/hcsshim v0.9.4 + github.com/aws/aws-sdk-go v1.44.116 github.com/containerd/continuity v0.3.0 github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 github.com/google/go-containerregistry v0.7.0 @@ -85,6 +86,7 @@ require ( github.com/rancher/wins v0.1.1 github.com/rancher/wrangler v0.8.11-0.20220217210408-3ecd23dfea3b github.com/sirupsen/logrus v1.9.0 + github.com/spf13/cobra v1.6.0 github.com/tigera/operator v1.28.1 github.com/urfave/cli v1.22.9 golang.org/x/sys v0.2.0 diff --git a/go.sum b/go.sum index d2916ec0d6..0653becb5a 100644 --- a/go.sum +++ b/go.sum @@ -211,8 +211,9 @@ github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= github.com/aws/aws-sdk-go v1.38.49/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.40.56/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= -github.com/aws/aws-sdk-go v1.44.24 h1:3nOkwJBJLiGBmJKWp3z0utyXuBkxyGkRRwWjrTItJaY= github.com/aws/aws-sdk-go v1.44.24/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.116 h1:NpLIhcvLWXJZAEwvPj3TDHeqp7DleK6ZUVYyW01WNHY= +github.com/aws/aws-sdk-go v1.44.116/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= @@ -376,8 +377,9 @@ github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= @@ -862,8 +864,9 @@ github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/influxdata/tdigest v0.0.1/go.mod h1:Z0kXnxzbTC2qrx4NaIzYkE1k66+6oEDQTvL95hQFh5Y= github.com/insomniacslk/dhcp v0.0.0-20220119180841-3c283ff8b7dd/go.mod h1:h+MxyHxRg9NH3terB1nfRIUaQEcI0XOVkdR9LNBlp8E= @@ -1519,8 +1522,9 @@ github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tL github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= +github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= diff --git a/tests/e2e/createreport/generate_report.go b/tests/e2e/createreport/generate_report.go new file mode 100644 index 0000000000..f63eda1c50 --- /dev/null +++ b/tests/e2e/createreport/generate_report.go @@ -0,0 +1,462 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "html/template" + "io/ioutil" + "log" + "math" + "net/http" + "os" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/spf13/cobra" +) + +// Output to store test results from specReport +type Output struct { + State string + Name string + Type string + Time float64 +} + +// GoTestJSONRowData to store logs in JSON format +type GoTestJSONRowData struct { + Time time.Time + Action string + Package string + Test string + Output string + Elapsed float64 +} + +// ProcessedTestdata to store formatted data +type ProcessedTestdata struct { + TotalTestTime string + TestDate string + FailedTests int + PassedTests int + SkippedTests int + TestSummary []TestOverview + TestSuiteSummary map[string]TestSuiteDetails +} + +// TestSuiteDetails to store test suite details +type TestSuiteDetails struct { + TestSuiteName string + ElapsedTime float64 + Status string + FailedTests int + PassedTests int + SkippedTests int +} + +// TestDetails to store test case details +type TestDetails struct { + TestSuiteName string + TestCaseName string + ElapsedTime float64 + Status string +} + +// TestOverview to store structured test case details per test suite +type TestOverview struct { + TestSuiteName string + TestCases []TestDetails +} + +func main() { + rootCmd := initCommand() + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +var fileName string +var testSuiteName string +var OS string +var htmlReport string +var bucketName string + +func initCommand() *cobra.Command { + var rootCmd = &cobra.Command{ + Use: "go-test-html-report", + Short: "go-test-html-report generates a html report of go-test logs", + RunE: func(cmd *cobra.Command, args []string) (e error) { + file, _ := cmd.Flags().GetString("file") + testData := make([]GoTestJSONRowData, 0) + if file != "" { + testData = ReadLogsFromFile(file) + } else { + log.Println("Log file not passed") + os.Exit(1) + } + OS = strings.Split(strings.Split(file, "_")[1], ".")[0] + processedTestdata := ProcessTestData(testData) + GenerateHTMLReport(processedTestdata.TotalTestTime, + processedTestdata.TestDate, + processedTestdata.TestSummary, + processedTestdata.TestSuiteSummary, + ) + log.Println("Report Generated") + LaunchHTML() + // upload test result + UploadReport(htmlReport) + // upload result result for all OS + UploadReport("launchresults.html") + return nil + }, + } + rootCmd.PersistentFlags().StringVarP( + &fileName, + "file", + "f", + "", + "set the file of the go test json logs") + return rootCmd +} + +// ReadLogsFromFile the logs generated from the test run script +func ReadLogsFromFile(fileName string) []GoTestJSONRowData { + file, err := os.Open(fileName) + if err != nil { + log.Println("error opening file: ", err) + os.Exit(1) + } + defer func() { + err := file.Close() + if err != nil { + log.Println("error closing file: ", err) + os.Exit(1) + } + }() + + scanner := bufio.NewScanner(file) + rowData := make([]GoTestJSONRowData, 0) + + for scanner.Scan() { + row := GoTestJSONRowData{} + + // unmarshall each line to GoTestJSONRowData + err := json.Unmarshal([]byte(scanner.Text()), &row) + if err != nil { + log.Println("error to unmarshall test logs: ", err) + os.Exit(1) + } + rowData = append(rowData, row) + } + + if err := scanner.Err(); err != nil { + log.Println("error with file scanner: ", err) + os.Exit(1) + } + return rowData +} + +// ProcessTestData reads json formatted data and stores into respective structures +func ProcessTestData(rowData []GoTestJSONRowData) ProcessedTestdata { + testDetails := map[string]TestDetails{} + testSuiteDetails := map[string]TestSuiteDetails{} + passedTests := 0 + failedTests := 0 + skippedTests := 0 + // Loop through logs + for _, r := range rowData { + if r.Test != "" { + testSuiteName = r.Test + var jsonMap Output + + // Extract valid data from the logs + if strings.Contains(r.Output, "k3s test") || strings.Contains(r.Output, "rke2 test") { + output2 := strings.LastIndex(r.Output, "}") + output2 = output2 + 1 + json.Unmarshal([]byte(strings.TrimSpace(r.Output[:output2])), &jsonMap) + } + + // Check if there is a valid test case + if len(jsonMap.Name) > 1 { + if jsonMap.State == "failed" || jsonMap.State == "passed" || jsonMap.State == "skipped" { + testDetails[r.Test+jsonMap.Name] = TestDetails{ + TestSuiteName: r.Test, + TestCaseName: jsonMap.Name, + ElapsedTime: jsonMap.Time / (1000 * 1000 * 1000 * 60), + Status: jsonMap.State, + } + } + + if jsonMap.State == "failed" { + failedTests = failedTests + 1 + } else if jsonMap.State == "passed" { + passedTests = passedTests + 1 + } else if jsonMap.State == "skipped" { + skippedTests = skippedTests + 1 + } + } + } else { + if r.Action == "fail" || r.Action == "pass" || r.Action == "skip" { + testSuiteDetails[testSuiteName] = TestSuiteDetails{ + TestSuiteName: testSuiteName, + ElapsedTime: r.Elapsed / 60, + Status: r.Action, + FailedTests: failedTests, + PassedTests: passedTests, + SkippedTests: skippedTests, + } + passedTests = 0 + failedTests = 0 + skippedTests = 0 + + } + } + } + + testSummary := make([]TestOverview, 0) + for key := range testSuiteDetails { + testCases := make([]TestDetails, 0) + for _, t2 := range testDetails { + if t2.TestSuiteName == key { + testCases = append(testCases, t2) + } + } + // testSummary holds testSuiteName and testCases + testSummary = append(testSummary, TestOverview{ + TestSuiteName: key, + TestCases: testCases, + }) + } + // determine total test time + totalTestTime := "" + if rowData[len(rowData)-1].Time.Sub(rowData[0].Time).Seconds() < 60 { + totalTestTime = fmt.Sprintf("%f s", rowData[len(rowData)-1].Time.Sub(rowData[0].Time).Seconds()) + } else { + min := int(math.Trunc(rowData[len(rowData)-1].Time.Sub(rowData[0].Time).Seconds() / 60)) + seconds := int(math.Trunc((rowData[len(rowData)-1].Time.Sub(rowData[0].Time).Minutes() - float64(min)) * 60)) + totalTestTime = fmt.Sprintf("%dm:%ds", min, seconds) + } + + // determine test date + testDate := rowData[0].Time.Format(time.RFC850) + + return ProcessedTestdata{ + TotalTestTime: totalTestTime, + TestDate: testDate, + FailedTests: failedTests, + PassedTests: passedTests, + SkippedTests: skippedTests, + TestSummary: testSummary, + TestSuiteSummary: testSuiteDetails, + } +} + +// GenerateHTMLReport generates report in the form rke2__results.html +func GenerateHTMLReport(totalTestTime, testDate string, testSummary []TestOverview, testSuiteDetails map[string]TestSuiteDetails) { + totalPassedTests := 0 + totalFailedTests := 0 + totalSkippedTests := 0 + templates := make([]template.HTML, 0) + for _, testSuite := range testSuiteDetails { + totalPassedTests = totalPassedTests + testSuite.PassedTests + totalFailedTests = totalFailedTests + testSuite.FailedTests + totalSkippedTests = totalSkippedTests + testSuite.SkippedTests + // display testSuiteName + htmlString := template.HTML("
\n") + packageInfoTemplateString := template.HTML("") + + packageInfoTemplateString = "
{{.testsuiteName}}
" + "\n" + "
Run Time: {{.elapsedTime}}m
" + "\n" + packageInfoTemplate, err := template.New("packageInfoTemplate").Parse(string(packageInfoTemplateString)) + if err != nil { + log.Println("error parsing package info template", err) + os.Exit(1) + } + + var processedPackageTemplate bytes.Buffer + err = packageInfoTemplate.Execute(&processedPackageTemplate, map[string]string{ + "testsuiteName": testSuite.TestSuiteName + "_" + OS, + "elapsedTime": fmt.Sprintf("%.2f", testSuite.ElapsedTime), + }) + if err != nil { + log.Println("error applying package info template: ", err) + os.Exit(1) + } + + if testSuite.Status == "pass" { + packageInfoTemplateString = "
" + + template.HTML(processedPackageTemplate.Bytes()) + "
" + } else if testSuite.Status == "fail" { + packageInfoTemplateString = "
" + + template.HTML(processedPackageTemplate.Bytes()) + "
" + } else { + packageInfoTemplateString = "
" + + template.HTML(processedPackageTemplate.Bytes()) + "
" + } + + htmlString = htmlString + "\n" + packageInfoTemplateString + testInfoTemplateString := template.HTML("") + + // display testCases + for _, pt := range testSummary { + testHTMLTemplateString := template.HTML("") + if len(pt.TestCases) == 0 { + log.Println("Test run failed for ", pt.TestSuiteName, "no testcases were executed") + continue + } + if pt.TestSuiteName == testSuite.TestSuiteName { + if testSuite.FailedTests == 0 { + testHTMLTemplateString = "
" + + "\n" + "
" + + "
+ {{.testName}}
" + "\n" + "
{{.elapsedTime}}
" + "\n" + + "
" + "\n" + + "
" + } else if testSuite.FailedTests > 0 { + testHTMLTemplateString = "
" + + "\n" + "
" + + "
+ {{.testName}}
" + "\n" + "
{{.elapsedTime}}
" + "\n" + + "
" + "\n" + + "
" + } else if testSuite.SkippedTests > 0 { + testHTMLTemplateString = "
" + + "\n" + "
" + + "
+ {{.testName}}
" + "\n" + "
{{.elapsedTime}}
" + "\n" + + "
" + "\n" + + "
" + } + testTemplate, err := template.New("Test").Parse(string(testHTMLTemplateString)) + if err != nil { + log.Println("error parsing tests template: ", err) + os.Exit(1) + } + var processedTestTemplate bytes.Buffer + err = testTemplate.Execute(&processedTestTemplate, map[string]string{ + "testName": "TestCases", + }) + if err != nil { + log.Println("error applying test template: ", err) + os.Exit(1) + } + + testHTMLTemplateString = template.HTML(processedTestTemplate.Bytes()) + testCaseHTMLTemplateString := template.HTML("") + + for _, tC := range pt.TestCases { + testCaseHTMLTemplateString = "
{{.testName}}
" + "\n" + "
{{.elapsedTime}}m
" + testCaseTemplate, err := template.New("testCase").Parse(string(testCaseHTMLTemplateString)) + if err != nil { + log.Println("error parsing test case template: ", err) + os.Exit(1) + } + + var processedTestCaseTemplate bytes.Buffer + err = testCaseTemplate.Execute(&processedTestCaseTemplate, map[string]string{ + "testName": tC.TestCaseName, + "elapsedTime": fmt.Sprintf("%f", tC.ElapsedTime), + }) + if err != nil { + log.Println("error applying test case template: ", err) + os.Exit(1) + } + + if tC.Status == "passed" { + testCaseHTMLTemplateString = "
" + template.HTML(processedTestCaseTemplate.Bytes()) + "
" + + } else if tC.Status == "failed" { + testCaseHTMLTemplateString = "
" + template.HTML(processedTestCaseTemplate.Bytes()) + "
" + + } else { + testCaseHTMLTemplateString = "
" + template.HTML(processedTestCaseTemplate.Bytes()) + "
" + } + testHTMLTemplateString = testHTMLTemplateString + "\n" + testCaseHTMLTemplateString + } + testHTMLTemplateString = testHTMLTemplateString + "\n" + "
" + "\n" + "
" + testInfoTemplateString = testInfoTemplateString + "\n" + testHTMLTemplateString + } + } + htmlString = htmlString + "\n" + "
\n" + testInfoTemplateString + "\n" + "
" + htmlString = htmlString + "\n" + "
" + templates = append(templates, htmlString) + } + reportTemplate := template.New("report-template.html") + reportTemplateData, err := Asset("report-template.html") + if err != nil { + log.Println("error retrieving report-template.html: ", err) + os.Exit(1) + } + + report, err := reportTemplate.Parse(string(reportTemplateData)) + if err != nil { + log.Println("error parsing report-template.html: ", err) + os.Exit(1) + } + + var processedTemplate bytes.Buffer + type templateData struct { + HTMLElements []template.HTML + FailedTests int + PassedTests int + SkippedTests int + TotalTestTime string + TestDate string + } + + err = report.Execute(&processedTemplate, + &templateData{ + HTMLElements: templates, + FailedTests: totalFailedTests, + PassedTests: totalPassedTests, + SkippedTests: totalSkippedTests, + TotalTestTime: totalTestTime, + TestDate: testDate, + }, + ) + if err != nil { + log.Println("error applying report-template.html: ", err) + os.Exit(1) + } + htmlReport = strings.Split(fileName, ".")[0] + "_results.html" + bucketName = strings.Split(htmlReport, "_")[0] + "-results" + fmt.Println(bucketName) + err = ioutil.WriteFile(htmlReport, processedTemplate.Bytes(), 0644) + if err != nil { + log.Println("error writing report.html file: ", err) + os.Exit(1) + } +} + +// UploadReport used to upload test result to s3 bucket +func UploadReport(filename string) { + file, err := os.Open(filename) + + if err != nil { + log.Println("error opening file: ", err) + os.Exit(1) + } + report1 := s3.New(session.New(), &aws.Config{Region: aws.String("us-east-2")}) + params1 := &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(filename), + ContentType: aws.String("text/html"), + Body: file, + } + _, err = report1.PutObject(params1) + if err != nil { + fmt.Println("Upload failed due to: ", err) + os.Exit(1) + } + + fmt.Println(filename, "is uploaded") + +} + +// LaunchHTML to reflect the changes dynamically to the matrix +func LaunchHTML() { + http.Handle("/", http.FileServer(http.Dir("./"))) + http.ListenAndServe(":80", nil) +} diff --git a/tests/e2e/createreport/launchresults.html b/tests/e2e/createreport/launchresults.html new file mode 100644 index 0000000000..5237ec227f --- /dev/null +++ b/tests/e2e/createreport/launchresults.html @@ -0,0 +1,34 @@ + + + + + + +

+

+

Ubuntu 20.04 Test Results

+ +

Rocky Linux8.7 Test Results

+ +

Fedora36 Test Results

+ +

openSUSE Leap 15.4 Test Results

+ + + diff --git a/tests/e2e/createreport/report-template-bindata.go b/tests/e2e/createreport/report-template-bindata.go new file mode 100644 index 0000000000..9fa636b2f5 --- /dev/null +++ b/tests/e2e/createreport/report-template-bindata.go @@ -0,0 +1,235 @@ +// Code generated by go-bindata. +// sources: +// report-template.html +// DO NOT EDIT! + +package main + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" +) + +func bindataRead(data []byte, name string) ([]byte, error) { + gz, err := gzip.NewReader(bytes.NewBuffer(data)) + if err != nil { + return nil, fmt.Errorf("Read %q: %v", name, err) + } + + var buf bytes.Buffer + _, err = io.Copy(&buf, gz) + clErr := gz.Close() + + if err != nil { + return nil, fmt.Errorf("Read %q: %v", name, err) + } + if clErr != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +type asset struct { + bytes []byte + info os.FileInfo +} + +type bindataFileInfo struct { + name string + size int64 + mode os.FileMode + modTime time.Time +} + +func (fi bindataFileInfo) Name() string { + return fi.name +} +func (fi bindataFileInfo) Size() int64 { + return fi.size +} +func (fi bindataFileInfo) Mode() os.FileMode { + return fi.mode +} +func (fi bindataFileInfo) ModTime() time.Time { + return fi.modTime +} +func (fi bindataFileInfo) IsDir() bool { + return false +} +func (fi bindataFileInfo) Sys() interface{} { + return nil +} + +var _reportTemplateHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xa4\x57\xdd\x6f\xdb\x36\x10\x7f\x9e\xff\x8a\x9b\xd6\xa1\x0e\x5a\xc9\x1f\xeb\x86\x42\x96\xfc\xd0\xb4\x45\x1f\xba\xb5\x58\xbc\x87\x61\xdd\x03\x2d\x9e\x65\x26\x14\x29\x90\x27\xc7\x5e\xe0\xff\x7d\xa0\x3e\x6c\x59\x56\xd2\xb4\x11\x10\x05\xe4\xdd\xef\x77\x1f\xbc\x3b\xca\xd1\x8f\x6f\x3f\x5d\x2e\xfe\xfe\xfc\x0e\xd6\x94\xc9\xf9\x20\x72\xff\x40\x32\x95\xc6\x1e\x2a\xcf\x6d\x20\xe3\xf3\x01\x00\x40\x94\x21\x31\x48\xd6\xcc\x58\xa4\xd8\xfb\x6b\xf1\xde\x7f\xed\xd5\x22\x12\x24\x71\xbe\x70\xef\x68\x54\x2d\x2a\x81\xa5\x9d\x44\xa0\x5d\x8e\xb1\x47\xb8\xa5\x51\x62\x6d\x0d\x72\x4f\x60\xb4\x26\xb8\x3b\xac\xdd\xb3\x64\xc9\x4d\x6a\x74\xa1\xb8\x9f\x68\xa9\x4d\x08\x3f\x4d\x7f\x99\x8e\x5f\x4d\x66\x27\x6a\xb5\xec\x76\x2d\x08\x8f\x92\xfd\xe0\xc8\x6d\x8b\x24\x41\x6b\xdf\x1c\xf8\x2e\x1d\xe4\xab\xd6\x38\x33\x37\xa9\x41\x54\xfd\xac\x2b\x26\xe4\xf7\x50\x1a\xe4\xf7\xb8\x79\x23\xf2\xef\xf4\x71\xd7\xcf\x98\xb3\xe4\x86\xa5\x78\xc9\x0c\xff\xc8\x76\xba\xe8\x66\x38\x35\x82\xfb\x84\x59\x2e\x19\xa1\xa3\x2c\x32\x65\x43\x98\xac\x0c\xb0\x82\xf4\xf1\x35\x3b\x87\x55\xda\x7e\xca\xf2\x10\x5e\xe7\xdb\x53\x0d\x2e\x6c\x2e\xd9\x2e\x2c\x55\xfb\x7d\x23\xb4\xf4\x24\xc7\x1e\x65\xd1\x3d\xb7\x82\xd3\x3a\x84\xc9\x78\xfc\x73\x27\x8e\x5e\xdf\x97\xda\x70\x34\xbe\x61\x5c\x14\x36\x84\x57\x5d\x79\xc6\x4c\x2a\x94\xbf\xd4\x44\x3a\x0b\xe1\xd7\xae\x3c\x67\x9c\x0b\x95\x76\x90\xed\xd0\x13\x2d\x25\xcb\xad\x58\x4a\xec\xc4\x9d\x14\xc6\xba\x63\xcd\xb5\x50\x84\xe6\xab\xf0\x0f\xc8\x9c\xad\x2e\x4b\x6f\x47\x9c\xf8\x76\x16\xf5\xbd\x49\xaa\xd2\x11\x82\xd2\xaa\x43\xe6\xda\xd8\x67\x52\xa4\x2a\x04\x89\x2b\x3a\x95\xea\x82\xa4\x50\xd8\x07\x5c\x69\x45\xbe\x15\xff\x61\x08\x93\xb3\xf4\x3d\x25\xfd\x0f\x67\x29\x64\x2b\xc2\x6e\x4b\x25\x5a\x11\x2a\x0a\xe1\xf9\x97\xf1\x78\xfa\xe6\x79\x3f\x19\x4b\x48\x6c\xf0\x61\x02\xef\xcb\x74\x3a\x99\x7a\x8f\xf5\xe6\xb2\xc2\x75\xd8\x0e\x07\x34\x86\xc9\xd9\x19\x65\x6c\xeb\xaf\x51\xa4\x6b\x0a\x61\xdc\xc9\xf6\x06\xcd\x4a\xea\xdb\x10\xd6\x82\xf3\xf6\xc8\x2a\x4f\xca\x30\x65\x05\x09\xad\xc2\x16\x09\x8c\x83\xa9\x05\x64\x16\x7d\x5d\xd0\xfd\x5d\x7a\x45\x8c\xec\xa7\x0d\x9a\x8d\xc0\x5b\xb8\x1b\xfc\x00\x0f\x77\x68\xf3\xf7\xe8\x26\x3d\x1d\x59\xd6\x22\x5f\xa0\x25\xdb\x49\x4d\xab\x6a\xb6\xbe\x64\x26\xc5\xde\x7b\xe0\x2b\x13\xfb\x49\xdc\x9d\xd1\x7d\x20\x76\x93\x3b\x7f\xaa\xd7\xbb\x2e\x75\x34\x2a\x2f\xcd\xf9\x20\x1a\x55\x97\x6f\xb4\xd4\x7c\x07\x89\x64\xd6\xc6\x9e\xbb\x30\xdd\xbd\xcc\xc5\x06\x4a\xbd\xd8\x3b\xa4\x77\x25\x71\x3b\x2b\xdf\x3e\x17\x06\x93\xea\xe4\xab\x43\x9a\xd5\x1d\x14\xc2\xe4\xb7\x7c\x3b\x83\xa6\xa0\x26\xe3\xf1\x66\xdd\x5c\xe3\x2d\xd2\xf3\x00\xbc\xb9\x0b\x14\xfe\xc4\x5c\x1b\x8a\x46\x5c\x6c\x1e\x44\xb5\x31\x6f\x19\x61\x08\x77\x77\x81\x5b\xb9\xc5\x7e\xdf\x25\xa8\xc3\x3b\x2b\xbc\xd6\xc7\x42\x94\x37\x66\xea\x61\x40\x3a\x77\x1d\xe1\x35\xe0\x56\x11\x79\xf3\xcf\xe5\x02\x1c\xa1\x2d\x6d\x7f\x3e\x4a\x9d\xf9\xfc\x1b\x88\x5b\x15\xe4\xcd\xdf\x97\x8b\x16\xf1\xfb\xa3\xf4\x5b\x89\xdb\x15\xe4\xcd\xaf\xaa\x55\x8b\xfa\xaa\x25\xbf\x97\xbb\xa7\xd6\xa0\x63\x6f\xbe\xd0\xc4\x64\x49\x0c\x24\xb2\xfa\x30\xdc\x9e\xa3\x5e\x88\x0c\x5b\xec\xad\xa3\xb9\xbb\x33\x4c\xa5\x08\xcf\x84\xe2\xb8\x7d\x09\xcf\x50\x62\xe6\xc6\x57\x18\x43\xf0\x61\xf1\xfb\xc7\x77\xd5\xda\xee\xf7\xb5\x7e\xa3\x71\xd8\x40\xc5\xf7\xfb\x41\xcd\x19\x8d\x5c\x2d\xcf\x07\x91\x4d\x8c\xc8\xa9\x32\x32\x1a\xc1\xb5\x85\x6a\x07\x48\x43\x62\x90\x11\x02\x53\xd0\x1a\x9f\xa5\xe6\x86\x99\x72\x0f\x62\xe0\x3a\x29\x9c\x9d\x20\x45\x6a\x9c\x78\xb3\xbb\x74\x69\xfd\x83\x65\x38\xf4\x5a\x58\xef\xa2\xea\xb1\x95\x36\x30\x94\x48\x20\x20\x86\xf1\x0c\x04\x44\x25\x5d\x20\x51\xa5\xb4\x9e\x81\x78\xf1\xe2\xa2\xd5\xc8\x8d\xb9\xce\xad\x1b\x43\xa1\x38\xae\x84\x42\x7e\x9c\x32\x07\xee\xeb\x8a\xfb\xba\xe6\xfe\x47\xfc\x1b\x24\x6b\x21\xb9\x41\x75\xb0\x73\x7d\x6a\xc7\x3d\x0e\xda\x24\x37\x3e\x47\x0a\xc2\x6c\x78\x7d\x71\x02\x11\x2b\x18\xd6\x90\x20\x69\x02\x0f\x84\x4a\x64\xc1\xd1\x9e\x64\xa0\x76\xdd\xbb\xe8\x9a\xad\x67\xd1\x79\x88\x35\xf1\x99\xf2\xd2\x20\xbb\x39\xd9\xdd\xf7\x0d\xdd\x73\xce\x80\x71\xfe\x6e\x83\x8a\x3e\x0a\x4b\xa8\xd0\x0c\xbd\x44\x8a\xe4\xc6\x7b\x09\xab\x42\x95\xb3\x0a\x86\x5d\xf7\x68\x2d\x6c\x15\x9b\x43\x05\xa4\xd3\x54\xe2\xd0\xab\xee\x65\xef\x34\x1d\xd5\x69\x55\xb7\x6b\x5c\x21\x15\x6e\x9b\xe2\xb8\x12\x4b\x29\x54\x3a\x3b\xcb\x60\x0d\x09\xca\x66\x0a\x32\xb6\xfd\x50\x4e\xc6\xfe\x44\xf5\xaa\x42\x0c\xaa\x90\xf2\x94\x7a\x0f\x28\x6d\xf7\x3b\xef\x61\x92\x5b\xa1\xb8\xbe\x0d\x84\x52\x68\xea\xcd\x17\xe0\xe5\x5b\x6f\x76\x5f\xbe\xeb\xba\x76\xed\xd5\x34\x54\x34\xaa\x7e\xc9\xfd\x1f\x00\x00\xff\xff\x46\x11\x73\x73\xda\x0d\x00\x00") + +func reportTemplateHtmlBytes() ([]byte, error) { + return bindataRead( + _reportTemplateHtml, + "report-template.html", + ) +} + +func reportTemplateHtml() (*asset, error) { + bytes, err := reportTemplateHtmlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "report-template.html", size: 3546, mode: os.FileMode(420), modTime: time.Unix(1673562223, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +// Asset loads and returns the asset for the given name. +// It returns an error if the asset could not be found or +// could not be loaded. +func Asset(name string) ([]byte, error) { + cannonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[cannonicalName]; ok { + a, err := f() + if err != nil { + return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err) + } + return a.bytes, nil + } + return nil, fmt.Errorf("Asset %s not found", name) +} + +// MustAsset is like Asset but panics when Asset would return an error. +// It simplifies safe initialization of global variables. +func MustAsset(name string) []byte { + a, err := Asset(name) + if err != nil { + panic("asset: Asset(" + name + "): " + err.Error()) + } + + return a +} + +// AssetInfo loads and returns the asset info for the given name. +// It returns an error if the asset could not be found or +// could not be loaded. +func AssetInfo(name string) (os.FileInfo, error) { + cannonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[cannonicalName]; ok { + a, err := f() + if err != nil { + return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err) + } + return a.info, nil + } + return nil, fmt.Errorf("AssetInfo %s not found", name) +} + +// AssetNames returns the names of the assets. +func AssetNames() []string { + names := make([]string, 0, len(_bindata)) + for name := range _bindata { + names = append(names, name) + } + return names +} + +// _bindata is a table, holding each asset generator, mapped to its name. +var _bindata = map[string]func() (*asset, error){ + "report-template.html": reportTemplateHtml, +} + +// AssetDir returns the file names below a certain +// directory embedded in the file by go-bindata. +// For example if you run go-bindata on data/... and data contains the +// following hierarchy: +// data/ +// foo.txt +// img/ +// a.png +// b.png +// then AssetDir("data") would return []string{"foo.txt", "img"} +// AssetDir("data/img") would return []string{"a.png", "b.png"} +// AssetDir("foo.txt") and AssetDir("notexist") would return an error +// AssetDir("") will return []string{"data"}. +func AssetDir(name string) ([]string, error) { + node := _bintree + if len(name) != 0 { + cannonicalName := strings.Replace(name, "\\", "/", -1) + pathList := strings.Split(cannonicalName, "/") + for _, p := range pathList { + node = node.Children[p] + if node == nil { + return nil, fmt.Errorf("Asset %s not found", name) + } + } + } + if node.Func != nil { + return nil, fmt.Errorf("Asset %s not found", name) + } + rv := make([]string, 0, len(node.Children)) + for childName := range node.Children { + rv = append(rv, childName) + } + return rv, nil +} + +type bintree struct { + Func func() (*asset, error) + Children map[string]*bintree +} +var _bintree = &bintree{nil, map[string]*bintree{ + "report-template.html": &bintree{reportTemplateHtml, map[string]*bintree{}}, +}} + +// RestoreAsset restores an asset under the given directory +func RestoreAsset(dir, name string) error { + data, err := Asset(name) + if err != nil { + return err + } + info, err := AssetInfo(name) + if err != nil { + return err + } + err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755)) + if err != nil { + return err + } + err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode()) + if err != nil { + return err + } + err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) + if err != nil { + return err + } + return nil +} + +// RestoreAssets restores an asset under the given directory recursively +func RestoreAssets(dir, name string) error { + children, err := AssetDir(name) + // File + if err != nil { + return RestoreAsset(dir, name) + } + // Dir + for _, child := range children { + err = RestoreAssets(dir, filepath.Join(name, child)) + if err != nil { + return err + } + } + return nil +} + +func _filePath(dir, name string) string { + cannonicalName := strings.Replace(name, "\\", "/", -1) + return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) +} + diff --git a/tests/e2e/createreport/report-template.html b/tests/e2e/createreport/report-template.html new file mode 100644 index 0000000000..dc86160ffd --- /dev/null +++ b/tests/e2e/createreport/report-template.html @@ -0,0 +1,131 @@ + + + + + Title + + + +
+
Test Report
+
Test Date: {{.TestDate}}
+
+

Passed tests: {{.PassedTests}}

+

Failed tests: {{.FailedTests}}

+

Skipped tests: {{.SkippedTests}}

+

Total test time: {{.TotalTestTime}}

+
+ {{range $index, $element := .HTMLElements}} + {{$element}} + {{end}} +
+ + + diff --git a/tests/e2e/scripts/run_tests.sh b/tests/e2e/scripts/run_tests.sh new file mode 100755 index 0000000000..24e9a3ca5e --- /dev/null +++ b/tests/e2e/scripts/run_tests.sh @@ -0,0 +1,62 @@ +#!/bin/bash +nodeOS=${1:-"generic/ubuntu2004"} +servercount=${2:-3} +agentcount=${3:-1} +db=${4:-"etcd"} +hardened=${5:-""} +rke2_version=${rke2_version} +rke2_channel=${rke2_channel:-"commit"} + +E2E_EXTERNAL_DB=$db && export E2E_EXTERNAL_DB +E2E_REGISTRY=true && export E2E_REGISTRY + +cd +cd rke2 && git pull --rebase origin master +/usr/local/go/bin/go mod tidy + +cd tests/e2e +OS=$(echo "$nodeOS"|cut -d'/' -f2) +echo "$OS" + +# create directory to store reports if it does not exists +if [ ! -d createreport ] +then + mkdir createreport +fi + +count=0 +run_tests(){ + count=$(( count + 1 )) + vagrant global-status | awk '/running/'|cut -c1-7| xargs -r -d '\n' -n 1 -- vagrant destroy -f + + E2E_RELEASE_VERSION=$rke2_version && export E2E_RELEASE_VERSION + E2E_RELEASE_CHANNEL=$rke2_channel && export E2E_RELEASE_CHANNEL + + echo 'RUNNING CLUSTER UPGRADE TEST' + E2E_REGISTRY=true /usr/local/go/bin/go test -v ./upgradecluster/upgradecluster_test.go -nodeOS="$nodeOS" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=1h -json -ci |tee createreport/rke2_"$OS".log + + echo 'RUNNING DUALSTACK VALIDATION TEST' + E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v dualstack/dualstack_test.go -nodeOS="$nodeOS" -serverCount=1 -agentCount=1 -timeout=30m -json -ci |tee -a createreport/rke2_"$OS".log + + echo 'RUNNING CLUSTER VALIDATION TEST' + E2E_REGISTRY=true E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v validatecluster/validatecluster_test.go -nodeOS="$nodeOS" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=30m -json -ci |tee -a createreport/rke2_"$OS".log + + echo 'RUNNING MIXEDOS TEST' + /usr/local/go/bin/go test -v mixedos/mixedos_test.go -nodeOS="$nodeOS" -serverCount=$((servercount)) -timeout=1h -json -ci |tee -a createreport/rke2_"$OS".log + + echo 'RUNNING SPLIT SERVER VALIDATION TEST' + E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v splitserver/splitserver_test.go -nodeOS="$nodeOS" -timeout=30m -json -ci |tee -a createreport/rke2_"$OS".log +} + +ls createreport/rke2_"$OS".log 2>/dev/null && rm createreport/rke2_"$OS".log +run_tests + +# re-run test if first run fails and keep record of repeatedly failed test to debug +while [ -f createreport/rke2_"$OS".log ] && grep -w ":fail" createreport/rke2_"$OS".log && [ $count -le 2 ] +do + cp createreport/rke2_"$OS".log createreport/rke2_"$OS"_"$count".log + run_tests +done + +# Generate report and upload to s3 bucket +cd createreport && /usr/local/go/bin/go run -v report-template-bindata.go generate_report.go -f rke2_"OS".log diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index 74efd9cff7..397fcac299 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -238,7 +238,7 @@ func RunCommand(cmd string) (string, error) { func UpgradeCluster(serverNodenames []string, agentNodenames []string) error { for _, nodeName := range serverNodenames { - cmd := "RELEASE_CHANNEL=commit vagrant provision " + nodeName + cmd := "E2E_RELEASE_CHANNEL=commit vagrant provision " + nodeName fmt.Println(cmd) if out, err := RunCommand(cmd); err != nil { fmt.Println("Error Upgrading Cluster", out) @@ -246,7 +246,7 @@ func UpgradeCluster(serverNodenames []string, agentNodenames []string) error { } } for _, nodeName := range agentNodenames { - cmd := "RELEASE_CHANNEL=commit vagrant provision " + nodeName + cmd := "E2E_RELEASE_CHANNEL=commit vagrant provision " + nodeName if _, err := RunCommand(cmd); err != nil { fmt.Println("Error Upgrading Cluster", err) return err diff --git a/tests/e2e/upgradecluster/Vagrantfile b/tests/e2e/upgradecluster/Vagrantfile index b9d496a821..af3b854d94 100644 --- a/tests/e2e/upgradecluster/Vagrantfile +++ b/tests/e2e/upgradecluster/Vagrantfile @@ -18,6 +18,7 @@ def provision(vm, roles, role_num, node_num) node_ip = "#{NETWORK_PREFIX}.#{100+node_num}" vm.network "private_network", ip: node_ip, netmask: "255.255.255.0" + scripts_location = Dir.exists?("./scripts/") ? "./scripts/" : "../scripts/" vagrant_defaults = '../vagrantdefaults.rb' load vagrant_defaults if File.exists?(vagrant_defaults)