fix: optimizations in git flows (#87)

* adding git commit message truncation logic for large messages

* first commit draft

* cleaning dead code

* added commit count

* fix git format in cmd

* added interfaces

* comments

* removing output

* minor restructuring

* removing deuplicacy for OpenNewRepo

* changes

* fixes

* wire

* old story refactor

* fixes for IT tests

* cleaning

* refactorings

* minor

* test fixes

* test fixes

* test fixes

* PR comments

* removing old changes

* cleaning up

* wire

* minor changes

* added comments on interface methods

* next error handling

* added timeout and context in one flow

* added timeouts

* remaning flows

* minor changes

* renaming to gitCtx

* added command timeout

* added command timeout

* moving cancel defer in poller

* renaming

* added command level timeouts

* fix

* minor

* removing process timeout

* refactoring

* cli for analytics

* changes

* changes

* minor

* adding gogit timeout

* debugging

* testing fix

* fixes

* fixes

* minor refactor

* pr comments

* fixed it tests

* common-lib upgrade

* common lib from main

* fixes-be

* adding support for last seen hash

* fix for history being overwritten

* fixing for empty history

* changes

pkg/RepoManages.go

@@ -194,12 +194,16 @@ func (impl RepoManagerImpl) updatePipelineMaterialCommit(gitCtx git.GitContext,
 			impl.logger.Errorw("error in fetching material", "err", err)
 			continue
 		}
-
-		commits, err := impl.repositoryManager.ChangesSince(gitCtx, material.CheckoutLocation, pipelineMaterial.Value, "", "", impl.configuration.GitHistoryCount)
+		fetchCount := impl.configuration.GitHistoryCount
+		commits, err := impl.repositoryManager.ChangesSince(gitCtx, material.CheckoutLocation, pipelineMaterial.Value, "", "", fetchCount)
 		//commits, err := impl.FetchChanges(pipelineMaterial.Id, "", "", 0)
 		if err == nil {
 			impl.logger.Infow("commits found", "commit", commits)
-			b, err := json.Marshal(commits)
+			totalCommits, err := git.AppendOldCommitsFromHistory(commits, pipelineMaterial.CommitHistory, fetchCount)
+			if err != nil {
+				impl.logger.Errorw("error in appending history to fetched commit", "err", err, "pipelineMaterialId", pipelineMaterial.Id)
+			}
+			b, err := json.Marshal(totalCommits)
 			if err == nil {
 				pipelineMaterial.CommitHistory = string(b)
 				if len(commits) > 0 {

pkg/git/Bean.go

@@ -17,6 +17,8 @@
 package git
 
 import (
+	"encoding/json"
+	"fmt"
 	"github.com/devtron-labs/git-sensor/internal/sql"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing/object"
@@ -141,6 +143,22 @@ type GitCommitBase struct {
 	Excluded    bool         `json:",omitempty"`
 }
 
+func AppendOldCommitsFromHistory(newCommits []*GitCommitBase, commitHistory string, fetchedCount int) ([]*GitCommitBase, error) {
+
+	oldCommits := make([]*GitCommitBase, 0)
+	if len(commitHistory) > 0 {
+		err := json.Unmarshal([]byte(commitHistory), &oldCommits)
+		if err != nil {
+			return newCommits, fmt.Errorf("unmarshalling error %v", err)
+		}
+	}
+	totalCommits := append(newCommits, oldCommits...)
+	if len(totalCommits) > fetchedCount {
+		totalCommits = totalCommits[:fetchedCount]
+	}
+	return totalCommits, nil
+}
+
 func (gitCommit *GitCommitBase) SetFileStats(stats *FileStats) {
 	gitCommit.FileStats = stats
 }

pkg/git/GitCliManager.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 )
 
 type GitCliManager interface {
@@ -175,11 +176,18 @@ func (impl *GitCliManagerImpl) processGitLogOutput(out string) ([]GitCommit, err
 
 	for _, formattedCommit := range gitCommitFormattedList {
 
+		subject := strings.TrimSpace(formattedCommit.Subject)
+		body := strings.TrimSpace(formattedCommit.Body)
+		message := subject
+		if len(body) > 0 {
+			message = strings.Join([]string{subject, body}, "\n")
+		}
+
 		cm := GitCommitBase{
 			Commit:  formattedCommit.Commit,
 			Author:  formattedCommit.Commiter.Name + " <" + formattedCommit.Commiter.Email + ">",
 			Date:    formattedCommit.Commiter.Date,
-			Message: formattedCommit.Subject + "\n" + formattedCommit.Body,
+			Message: message,
 		}
 		gitCommits = append(gitCommits, &GitCommitCli{
 			GitCommitBase: cm,

pkg/git/GitFormatter.go

@@ -2,7 +2,6 @@ package git
 
 import (
 	"encoding/json"
-	"fmt"
 	"strconv"
 	"strings"
 	"time"
@@ -17,8 +16,8 @@ var GITFORMAT = "--pretty=format:{" +
 	_dl_ + "commit" + _dl_ + ":" + _dl_ + "%H" + _dl_ + "," +
 	_dl_ + "parent" + _dl_ + ":" + _dl_ + "%P" + _dl_ + "," +
 	_dl_ + "refs" + _dl_ + ":" + _dl_ + "%D" + _dl_ + "," +
-	_dl_ + "subject" + _dl_ + ":" + _dl_ + "%s" + _dl_ + "," +
-	_dl_ + "body" + _dl_ + ":" + _dl_ + "%b" + _dl_ + "," +
+	_dl_ + "subject" + _dl_ + ":" + _dl_ + "%<(1024,trunc)%s" + _dl_ + "," +
+	_dl_ + "body" + _dl_ + ":" + _dl_ + "%<(1024,trunc)%b" + _dl_ + "," +
 	_dl_ + "author" + _dl_ +
 	":{" +
 	_dl_ + "name" + _dl_ + ":" + _dl_ + "%aN" + _dl_ + "," +
@@ -52,13 +51,16 @@ func parseFormattedLogOutput(out string) ([]GitCommitFormat, error) {
 	out = strings.ReplaceAll(out, "},\n", "},")
 
 	// to escape the special characters like quotes and newline characters in the commit data
-	logOut := strconv.Quote(out)
+	var sb strings.Builder
+	buffer := strconv.AppendQuote(make([]byte, 0, len(out)), out)
+	sb.Write(buffer)
+	logOut := sb.String()
 
 	//replace the delimiter with quotes to make it parsable json
 	logOut = strings.ReplaceAll(logOut, _dl_, `"`)
 
-	logOut = logOut[1 : len(logOut)-2]   // trim surround characters (surrounding quotes and trailing comma)
-	logOut = fmt.Sprintf("[%s]", logOut) // Add []
+	logOut = logOut[1 : len(logOut)-2] // trim surround characters (surrounding quotes and trailing comma)
+	logOut = strings.Join([]string{"[", "]"}, logOut)
 
 	var gitCommitFormattedList []GitCommitFormat
 	err := json.Unmarshal([]byte(logOut), &gitCommitFormattedList)
@@ -85,7 +87,7 @@ func (formattedCommit GitCommitFormat) transformToCommit() *Commit {
 		},
 		Tag:     &Tag{},
 		Tree:    &Tree{},
-		Subject: formattedCommit.Subject,
-		Body:    formattedCommit.Body,
+		Subject: strings.TrimSpace(formattedCommit.Subject),
+		Body:    strings.TrimSpace(formattedCommit.Body),
 	}
 }

pkg/git/Watcher.go

@@ -232,14 +232,26 @@ func (impl GitWatcherImpl) pollGitMaterialAndNotify(material *sql.GitMaterial) e
 		impl.logger.Debugw("Running changesBySinceRepository for material - ", material)
 		impl.logger.Debugw("---------------------------------------------------------- ")
 		// parse env variables here, then search for the count field and pass here.
-		commits, err := impl.repositoryManager.ChangesSinceByRepository(gitCtx, repo, material.Value, "", "", impl.configuration.GitHistoryCount)
+		lastSeenHash := ""
+		if len(material.LastSeenHash) > 0 {
+			// this might misbehave is the hash stored in table is corrupted somehow
+			lastSeenHash = material.LastSeenHash
+		}
+		fetchCount := impl.configuration.GitHistoryCount
+		commits, err := impl.repositoryManager.ChangesSinceByRepository(gitCtx, repo, material.Value, lastSeenHash, "", fetchCount)
 		if err != nil {
 			material.Errored = true
 			material.ErrorMsg = err.Error()
 			erroredMaterialsModels = append(erroredMaterialsModels, material)
 		} else if len(commits) > 0 {
 			latestCommit := commits[0]
 			if latestCommit.GetCommit().Commit != material.LastSeenHash {
+
+				commitsTotal, err := AppendOldCommitsFromHistory(commits, material.CommitHistory, fetchCount)
+				if err != nil {
+					impl.logger.Errorw("error in appending history to new commits", "material", material.GitMaterialId, "err", err)
+				}
+
 				// new commit found
 				mb := &CiPipelineMaterialBean{
 					Id:            material.Id,
@@ -255,7 +267,7 @@ func (impl GitWatcherImpl) pollGitMaterialAndNotify(material *sql.GitMaterial) e
 				material.CommitAuthor = latestCommit.Author
 				material.CommitDate = latestCommit.Date
 				material.CommitMessage = latestCommit.Message
-				commitJson, _ := json.Marshal(commits)
+				commitJson, _ := json.Marshal(commitsTotal)
 				material.CommitHistory = string(commitJson)
 				material.Errored = false
 				material.ErrorMsg = ""