AD-105: add frequency of ci-fail issues (#119)

AD-105

backend/api/apis/failure/v1alpha1/failure.go

@@ -0,0 +1,10 @@
+package v1alpha1
+
+type Failure struct {
+	JiraKey      string  `json:"jira_key"`
+	JiraStatus   string  `json:"jira_status"`
+	ErrorMessage string  `json:"error_message"`
+	Frequency    float64 `json:"frequency"`
+}
+
+type Failures []Failure

backend/api/apis/prow/v1alpha1/jobs.go

@@ -42,4 +42,7 @@ type Job struct {
 
 	// URL to the e2e-report.xml of the job
 	SuitesXmlUrl string `json:"suites_xml_url"`
+
+	// Set of error logs from build-log.txt
+	BuildErrorLogs string `json:"build_error_logs"`
 }

backend/api/server/failure_rotate.go

@@ -0,0 +1,29 @@
+package server
+
+import (
+	failureV1Alpha1 "github.com/redhat-appstudio/quality-studio/api/apis/failure/v1alpha1"
+)
+
+func (s *Server) UpdateFailuresByTeam() {
+	teamArr, _ := s.cfg.Storage.GetAllTeamsFromDB()
+
+	for _, team := range teamArr {
+		failures, _ := s.cfg.Storage.GetAllFailures(team)
+
+		for _, failure := range failures {
+			jiraStatus, err := s.cfg.Storage.GetJiraStatus(failure.JiraKey)
+			if err != nil {
+				s.cfg.Logger.Sugar().Warnf("Failed to get jira status:", err)
+			}
+
+			err = s.cfg.Storage.CreateFailure(failureV1Alpha1.Failure{
+				JiraKey:      failure.JiraKey,
+				JiraStatus:   jiraStatus,
+				ErrorMessage: failure.ErrorMessage,
+			}, team.ID)
+			if err != nil {
+				s.cfg.Logger.Sugar().Warnf("Failed to update failures:", err)
+			}
+		}
+	}
+}

backend/api/server/prow_rotate.go

@@ -7,10 +7,12 @@ import (
 	"io/ioutil"
 	"net/http"
 	"strings"
+	"time"
 
 	repoV1Alpha1 "github.com/redhat-appstudio/quality-studio/api/apis/github/v1alpha1"
 	prowV1Alpha1 "github.com/redhat-appstudio/quality-studio/api/apis/prow/v1alpha1"
 	"github.com/redhat-appstudio/quality-studio/api/server/router/prow"
+	"github.com/redhat-appstudio/quality-studio/pkg/constants"
 	"github.com/redhat-appstudio/quality-studio/pkg/storage"
 )
 
@@ -34,6 +36,31 @@ func (s *Server) UpdateProwStatusByTeam() {
 		repo, _ := s.cfg.Storage.ListRepositories(team)
 		s.ProwStaticUpdate(repo, prowjobs)
 	}
+
+	s.BuildLogErrorsUpdate()
+}
+
+// temporary func to update build log error messages from last 2 months saved prow jobs
+func (s *Server) BuildLogErrorsUpdate() {
+	startDate := time.Now().AddDate(0, -2, 0).Format(constants.DateFormat)
+	endDate := time.Now().Format(constants.DateFormat)
+
+	prowJobs, err := s.cfg.Storage.GetAllProwJobs(startDate, endDate)
+	if err != nil {
+		s.cfg.Logger.Sugar().Error("Failed to get all prow jobs ", err)
+	}
+
+	for _, pj := range prowJobs {
+		if *pj.E2eFailedTestMessages == "" &&
+			pj.State == string(prow.FailureState) &&
+			pj.BuildErrorLogs == nil {
+			buildErrors := getBuildLogErrors(pj.JobURL)
+			if buildErrors != "" {
+				s.cfg.Storage.UpdateBuildLogErrors(pj.JobID, buildErrors)
+			}
+		}
+	}
+
 }
 
 func (s *Server) ProwStaticUpdate(storageRepos []repoV1Alpha1.Repository, prowjobs []prow.ProwJob) {
@@ -46,8 +73,9 @@ func (s *Server) ProwStaticUpdate(storageRepos []repoV1Alpha1.Repository, prowjo
 			if prowOrg == repo.Owner.Login && prowRepo == repo.Name && pj.Status.State != prow.AbortedState && pj.Status.State != prow.PendingState && !strings.Contains(pj.Status.URL, "-images") && !strings.Contains(pj.Status.URL, "-index") {
 				// check if job already in database
 				prowJobsInDatabase, _ := s.cfg.Storage.GetProwJobsResultsByJobID(pj.Status.BuildID)
+
 				if len(prowJobsInDatabase) > 0 {
-					s.cfg.Logger.Sugar().Debugf("Data already exist in database about jobID %v, %v", pj.Status.BuildID)
+					s.cfg.Logger.Sugar().Debugf("Data already exists in database about jobID %v, %v", pj.Status.BuildID)
 					continue
 				}
 
@@ -64,6 +92,7 @@ func (s *Server) ProwStaticUpdate(storageRepos []repoV1Alpha1.Repository, prowjo
 						s.cfg.Logger.Sugar().Warnf("Failed convert xml file to golang bytes ", err)
 					}
 				}
+
 				if err := SaveProwJobsinDatabase(s.cfg.Storage, pj, suitesXml, repo.ID, suitesXmlUrl); err != nil {
 					s.cfg.Logger.Sugar().Error("Failed to save job database ", err)
 				}
@@ -72,9 +101,31 @@ func (s *Server) ProwStaticUpdate(storageRepos []repoV1Alpha1.Repository, prowjo
 	}
 }
 
+func getBuildLogErrors(url string) string {
+	buildErrorLogs := ""
+	buildFileUrl := url + "/" + "build-log.txt"
+	buildContent, _ := fetchSuitesXml(buildFileUrl)
+	buildErrorLogs = string(buildContent)
+
+	// keep last 50 lines
+	lines := strings.Split(buildErrorLogs, "\n")
+	if len(lines) > 50 {
+		lastIdx := len(lines) - 1
+		firstIdx := lastIdx - 50
+		lines = lines[firstIdx:lastIdx]
+		buildErrorLogs = strings.Join(lines, "\n")
+	}
+	return buildErrorLogs
+}
+
 func SaveProwJobsinDatabase(s storage.Storage, pj prow.ProwJob, ts prow.TestSuites, repositoryId, suitesXmlUrl string) error {
 	prowJob := prowV1Alpha1.Job{}
 	testSuiteSummary := getSuitesData(pj, ts)
+	buildErrorLogs := ""
+
+	if testSuiteSummary.E2EFailedMessages == "" && pj.Status.State == prow.FailureState {
+		buildErrorLogs = getBuildLogErrors(pj.Status.URL)
+	}
 
 	prowJob.JobID = pj.Status.BuildID
 	prowJob.CreatedAt = pj.Status.StartTime
@@ -89,6 +140,7 @@ func SaveProwJobsinDatabase(s storage.Storage, pj prow.ProwJob, ts prow.TestSuit
 	prowJob.CIFailed = getCIFailed(pj, ts)
 	prowJob.E2EFailedTestMessages = testSuiteSummary.E2EFailedMessages
 	prowJob.SuitesXmlUrl = suitesXmlUrl
+	prowJob.BuildErrorLogs = buildErrorLogs
 
 	if err := s.CreateProwJobResults(prowJob, repositoryId); err != nil {
 		return fmt.Errorf("failed to save job to db %s", err)

backend/api/server/rotate.go

@@ -56,6 +56,7 @@ func (s *Server) rotate() error {
 	}
 
 	s.UpdateProwStatusByTeam()
+	s.UpdateFailuresByTeam()
 	err = s.UpdateDataBaseRepoByTeam()
 	if err != nil {
 		s.cfg.Logger.Sugar().Errorf("Failed to update cache", zap.Error(err))
@@ -66,7 +67,7 @@ func (s *Server) rotate() error {
 }
 func staticRotationStrategy() rotationStrategy {
 	return rotationStrategy{
-		rotationFrequency: time.Minute * 15,
+		rotationFrequency: time.Minute * 2,
 	}
 }
 

backend/api/server/router/failure/failure.go

@@ -0,0 +1,119 @@
+package failure
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+
+	failureV1Alpha1 "github.com/redhat-appstudio/quality-studio/api/apis/failure/v1alpha1"
+	"github.com/redhat-appstudio/quality-studio/api/types"
+	"github.com/redhat-appstudio/quality-studio/pkg/utils/httputils"
+	"go.uber.org/zap"
+)
+
+func (f *failureRouter) createFailure(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+	var fr FailureRequest
+	if err := json.NewDecoder(r.Body).Decode(&fr); err != nil {
+		return httputils.WriteJSON(w, http.StatusInternalServerError, &types.ErrorResponse{
+			Message:    "Error reading team/error_message/jira_key value from body",
+			StatusCode: http.StatusBadRequest,
+		})
+	}
+
+	team, err := f.Storage.GetTeamByName(fr.Team)
+	if err != nil {
+		f.Logger.Error("Failed to fetch team. Make sure the team exists", zap.String("team", fr.Team), zap.Error(err))
+
+		return httputils.WriteJSON(w, http.StatusInternalServerError, &types.ErrorResponse{
+			Message:    err.Error(),
+			StatusCode: http.StatusBadRequest,
+		})
+	}
+
+	jiraStatus, err := f.Storage.GetJiraStatus(fr.JiraKey)
+	if err != nil {
+		f.Logger.Sugar().Warnf("Failed to get jira status:", err)
+	}
+
+	err = f.Storage.CreateFailure(failureV1Alpha1.Failure{
+		JiraKey:      fr.JiraKey,
+		JiraStatus:   jiraStatus,
+		ErrorMessage: fr.ErrorMessage,
+	}, team.ID)
+	if err != nil {
+		return httputils.WriteJSON(w, http.StatusInternalServerError, &types.ErrorResponse{
+			Message:    "Failed to save failure data in database.",
+			StatusCode: http.StatusBadRequest,
+		})
+	}
+
+	return httputils.WriteJSON(w, http.StatusOK, types.SuccessResponse{
+		Message:    "Successfully created failure in quality-studio",
+		StatusCode: http.StatusCreated,
+	})
+}
+
+func (f *failureRouter) getFailures(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+	teamName := r.URL.Query()["team_name"]
+	startDate := r.URL.Query()["start_date"]
+	endDate := r.URL.Query()["end_date"]
+
+	if len(teamName) == 0 {
+		return httputils.WriteJSON(w, http.StatusBadRequest, types.ErrorResponse{
+			Message:    "team_name value not present in query",
+			StatusCode: 400,
+		})
+	} else if len(startDate) == 0 {
+		return httputils.WriteJSON(w, http.StatusBadRequest, types.ErrorResponse{
+			Message:    "start_date value not present in query",
+			StatusCode: 400,
+		})
+	} else if len(endDate) == 0 {
+		return httputils.WriteJSON(w, http.StatusBadRequest, types.ErrorResponse{
+			Message:    "end_date value not present in query",
+			StatusCode: 400,
+		})
+	}
+
+	team, err := f.Storage.GetTeamByName(teamName[0])
+	if err != nil {
+		f.Logger.Error("Failed to get team")
+
+		return httputils.WriteJSON(w, http.StatusInternalServerError, &types.ErrorResponse{
+			Message:    err.Error(),
+			StatusCode: http.StatusBadRequest,
+		})
+	}
+
+	failures, err := f.Storage.GetFailuresByDate(team, startDate[0], endDate[0])
+	if err != nil {
+		return httputils.WriteJSON(w, http.StatusBadRequest, types.ErrorResponse{
+			Message:    "Failed to get failures by team.",
+			StatusCode: 400,
+		})
+	}
+
+	return httputils.WriteJSON(w, http.StatusOK, failures)
+}
+
+func (f *failureRouter) deleteFailure(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+	var fr FailureRequest
+	if err := json.NewDecoder(r.Body).Decode(&fr); err != nil {
+		return httputils.WriteJSON(w, http.StatusInternalServerError, &types.ErrorResponse{
+			Message:    "Error reading team/error_message/jira_key value from body",
+			StatusCode: http.StatusBadRequest,
+		})
+	}
+
+	err := f.Storage.DeleteFailure(fr.JiraKey)
+	if err != nil {
+		return httputils.WriteJSON(w, http.StatusBadRequest, types.ErrorResponse{
+			Message:    "Failed to delete failure",
+			StatusCode: 400,
+		})
+	}
+
+	return httputils.WriteJSON(w, http.StatusOK, types.SuccessResponse{
+		Message: "Failure deleted",
+	})
+}

backend/api/server/router/failure/failure_route.go

@@ -0,0 +1,37 @@
+package failure
+
+import (
+	"github.com/redhat-appstudio/quality-studio/api/server/router"
+	"github.com/redhat-appstudio/quality-studio/pkg/logger"
+	"github.com/redhat-appstudio/quality-studio/pkg/storage"
+	"go.uber.org/zap"
+)
+
+// failureRouter provides information about the server version.
+type failureRouter struct {
+	Route   []router.Route
+	Storage storage.Storage
+	Logger  *zap.Logger
+}
+
+// NewRouter initializes a new system router
+func NewRouter(s storage.Storage) router.Router {
+	logger, _ := logger.InitZap("info")
+	r := &failureRouter{}
+
+	r.Storage = s
+	r.Logger = logger
+
+	r.Route = []router.Route{
+		router.NewPostRoute("/failures/create", r.createFailure),
+		router.NewGetRoute("/failures/get", r.getFailures),
+		router.NewDeleteRoute("/failures/delete", r.deleteFailure),
+	}
+
+	return r
+}
+
+// Routes returns all the API routes dedicated to the server system
+func (s *failureRouter) Routes() []router.Route {
+	return s.Route
+}

backend/api/server/router/failure/types.go

@@ -0,0 +1,7 @@
+package failure
+
+type FailureRequest struct {
+	Team         string `json:"team"`
+	ErrorMessage string `json:"error_message"`
+	JiraKey      string `json:"jira_key"`
+}

backend/api/server/router/jira/jira.go

@@ -251,3 +251,40 @@ func (s *jiraRouter) getJiraProjects(ctx context.Context, w http.ResponseWriter,
 
 	return httputils.WriteJSON(w, http.StatusOK, projects)
 }
+
+func (s *jiraRouter) bugExists(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+	teamName := r.URL.Query()["team_name"]
+	key := r.URL.Query()["jira_key"]
+
+	if len(teamName) == 0 {
+		return httputils.WriteJSON(w, http.StatusBadRequest, types.ErrorResponse{
+			Message:    "team_name value not present in query",
+			StatusCode: 400,
+		})
+	} else if len(key) == 0 {
+		return httputils.WriteJSON(w, http.StatusBadRequest, types.ErrorResponse{
+			Message:    "jira_key value not present in query",
+			StatusCode: 400,
+		})
+	}
+
+	team, err := s.Storage.GetTeamByName(teamName[0])
+	if err != nil {
+		return httputils.WriteJSON(w, http.StatusInternalServerError, &types.ErrorResponse{
+			Message:    err.Error(),
+			StatusCode: http.StatusBadRequest,
+		})
+	}
+
+	exists, err := s.Storage.BugExists(key[0], team)
+	if err != nil {
+		s.Logger.Error("Failed to check if jira key exists")
+
+		return httputils.WriteJSON(w, http.StatusInternalServerError, &types.ErrorResponse{
+			Message:    err.Error(),
+			StatusCode: http.StatusBadRequest,
+		})
+	}
+
+	return httputils.WriteJSON(w, http.StatusOK, exists)
+}

backend/api/server/router/jira/jira_route.go

@@ -33,6 +33,7 @@ func NewRouter(s storage.Storage) router.Router {
 		router.NewGetRoute("/jira/project/list", r.getJiraProjects),
 		router.NewPostRoute("/jira/bugs/metrics/resolution", r.calculateRates),
 		router.NewPostRoute("/jira/bugs/metrics/open", r.openBugsMetrics),
+		router.NewGetRoute("/jira/bugs/exist", r.bugExists),
 	}
 
 	return r