Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactored Git Work #425

Merged
merged 1 commit into from
Apr 11, 2024
Merged

Refactored Git Work #425

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 13 additions & 92 deletions server/modules/elastalert/elastalert.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,15 @@ import (
"io"
"io/fs"
"net/http"
"net/url"
"os"
"os/exec"
"path"
"path/filepath"
"reflect"
"regexp"
"strings"
"sync"
"time"

"github.com/go-git/go-git/v5"
"github.com/samber/lo"
"github.com/security-onion-solutions/securityonion-soc/model"
"github.com/security-onion-solutions/securityonion-soc/module"
Expand Down Expand Up @@ -416,7 +413,7 @@ func (e *ElastAlertEngine) startCommunityRuleImport() {
templateFound = true
}

var allRepos map[string]*module.RuleRepo
allRepos := map[string]*module.RuleRepo{}
var repoChanges bool

var zips map[string][]byte
Expand All @@ -436,9 +433,15 @@ func (e *ElastAlertEngine) startCommunityRuleImport() {
continue
}

allRepos, repoChanges, err = e.updateRepos()
var dirtyRepos map[string]*mutil.DirtyRepo

dirtyRepos, repoChanges, err = mutil.UpdateRepos(&e.isRunning, e.reposFolder, e.rulesRepos)
if err != nil {
log.WithError(err).Error("unable to update sigma repos")
if strings.Contains(err.Error(), "module stopped") {
break
}

log.WithError(err).Error("unable to update Sigma repos")

if e.notify {
e.srv.Host.Broadcast("detection-sync", "detection", server.SyncStatus{
Expand All @@ -449,6 +452,10 @@ func (e *ElastAlertEngine) startCommunityRuleImport() {

continue
}

for k, v := range dirtyRepos {
allRepos[k] = v.Repo
}
} else {
// Possible airgap installation, or admin has disabled auto-updates.

Expand Down Expand Up @@ -585,92 +592,6 @@ func (e *ElastAlertEngine) startCommunityRuleImport() {
}
}

func (e *ElastAlertEngine) updateRepos() (allRepos map[string]*module.RuleRepo, anythingNew bool, err error) {
allRepos = map[string]*module.RuleRepo{} // map[repoPath]repo

// read existing repos
entries, err := os.ReadDir(e.reposFolder)
if err != nil {
log.WithError(err).Error("Failed to read sigma repos folder")
return nil, false, err
}

existingRepos := map[string]struct{}{}

for _, entry := range entries {
if !entry.IsDir() {
continue
}

existingRepos[entry.Name()] = struct{}{}
}

// pull or clone repos
for _, repo := range e.rulesRepos {
if !e.isRunning {
return nil, false, errModuleStopped
}

parser, err := url.Parse(repo.Repo)
if err != nil {
log.WithError(err).WithField("repo", repo).Error("Failed to parse repo URL, doing nothing with it")
continue
}

_, lastFolder := path.Split(parser.Path)
repoPath := filepath.Join(e.reposFolder, lastFolder)

allRepos[repoPath] = repo

if _, ok := existingRepos[lastFolder]; ok {
// repo already exists, pull
gitrepo, err := git.PlainOpen(repoPath)
if err != nil {
log.WithError(err).WithField("repo", gitrepo).Error("Failed to open repo, doing nothing with it")
continue
}

work, err := gitrepo.Worktree()
if err != nil {
log.WithError(err).WithField("repo", gitrepo).Error("Failed to get worktree, doing nothing with it")
continue
}

ctx, cancel := context.WithTimeout(e.srv.Context, time.Minute*5)

err = work.PullContext(ctx, &git.PullOptions{
Depth: 1,
SingleBranch: true,
})
if err != nil && err != git.NoErrAlreadyUpToDate {
cancel()
log.WithError(err).WithField("repo", repo).Error("Failed to pull repo, doing nothing with it")
continue
}
cancel()

if err == nil {
anythingNew = true
}
} else {
// repo does not exist, clone
_, err = git.PlainClone(repoPath, false, &git.CloneOptions{
Depth: 1,
SingleBranch: true,
URL: repo.Repo,
})
if err != nil {
log.WithError(err).WithField("repo", repo).Error("Failed to clone repo, doing nothing with it")
continue
}

anythingNew = true
}
}

return allRepos, anythingNew, nil
}

func (e *ElastAlertEngine) parseZipRules(pkgZips map[string][]byte) (detections []*model.Detection, errMap map[string]error) {
errMap = map[string]error{} // map[pkgName|fileName]error
defer func() {
Expand Down
115 changes: 19 additions & 96 deletions server/modules/strelka/strelka.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,8 @@ import (
"fmt"
"io/fs"
"net/http"
"net/url"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"strings"
Expand All @@ -29,7 +27,6 @@ import (
"github.com/security-onion-solutions/securityonion-soc/util"

"github.com/apex/log"
"github.com/go-git/go-git/v5"
)

var errModuleStopped = fmt.Errorf("strelka module has stopped running")
Expand Down Expand Up @@ -267,92 +264,34 @@ func (e *StrelkaEngine) startCommunityRuleImport() {
templateFound = true
}

// read existing repos
entries, err := os.ReadDir(e.reposFolder)
if err != nil {
log.WithError(err).Error("Failed to read yara repos folder")

if e.notify {
e.srv.Host.Broadcast("detection-sync", "detection", server.SyncStatus{
Engine: model.EngineNameStrelka,
Status: "error",
})
}

continue
}

existingRepos := map[string]struct{}{}

for _, entry := range entries {
if !entry.IsDir() {
continue
}

existingRepos[entry.Name()] = struct{}{}
}

upToDate := map[string]*module.RuleRepo{}

if e.autoUpdateEnabled {
// pull or clone repos
for _, repo := range e.rulesRepos {
if !e.isRunning {
allRepos, anythingNew, err := mutil.UpdateRepos(&e.isRunning, e.reposFolder, e.rulesRepos)
if err != nil {
if strings.Contains(err.Error(), "module stopped") {
break
}
}
if !anythingNew && !forceSync {
// no updates, skip
log.Info("Strelka sync found no changes")

parser, err := url.Parse(repo.Repo)
if err != nil {
log.WithError(err).WithField("repo", repo).Error("Failed to parse repo URL, doing nothing with it")
continue
}

_, lastFolder := path.Split(parser.Path)
repoPath := filepath.Join(e.reposFolder, lastFolder)

if _, ok := existingRepos[lastFolder]; ok {
// repo already exists, pull
gitrepo, err := git.PlainOpen(repoPath)
if err != nil {
log.WithError(err).WithField("repo", gitrepo).Error("Failed to open repo, doing nothing with it")
continue
}

work, err := gitrepo.Worktree()
if err != nil {
log.WithError(err).WithField("repo", gitrepo).Error("Failed to get worktree, doing nothing with it")
continue
}

ctx, cancel := context.WithTimeout(e.srv.Context, time.Minute*5)
mutil.WriteStateFile(e.IOManager, e.stateFilePath)

err = work.PullContext(ctx, &git.PullOptions{
Depth: 1,
SingleBranch: true,
if e.notify {
e.srv.Host.Broadcast("detection-sync", "detection", server.SyncStatus{
Engine: model.EngineNameStrelka,
Status: "success",
})
if err != nil && err != git.NoErrAlreadyUpToDate {
cancel()
log.WithError(err).WithField("repo", repo).Error("Failed to pull repo, doing nothing with it")
continue
}
cancel()
}

if err == nil || forceSync {
upToDate[repoPath] = repo
}
} else {
// repo does not exist, clone
_, err = git.PlainClone(repoPath, false, &git.CloneOptions{
Depth: 1,
SingleBranch: true,
URL: repo.Repo,
})
if err != nil {
log.WithError(err).WithField("repo", repo).Error("Failed to clone repo, doing nothing with it")
continue
}
continue
}

upToDate[repoPath] = repo
for k, v := range allRepos {
if v.WasModified {
upToDate[k] = v.Repo
}
}
} else {
Expand All @@ -364,22 +303,6 @@ func (e *StrelkaEngine) startCommunityRuleImport() {
return
}

if len(upToDate) == 0 {
// no updates, skip
log.Info("Strelka sync found no changes")

mutil.WriteStateFile(e.IOManager, e.stateFilePath)

if e.notify {
e.srv.Host.Broadcast("detection-sync", "detection", server.SyncStatus{
Engine: model.EngineNameStrelka,
Status: "success",
})
}

continue
}

communityDetections, err := e.srv.Detectionstore.GetAllCommunitySIDs(e.srv.Context, util.Ptr(model.EngineNameStrelka))
if err != nil {
log.WithError(err).Error("Failed to get all community SIDs")
Expand Down Expand Up @@ -726,7 +649,7 @@ func (e *StrelkaEngine) syncDetections(ctx context.Context) (errMap map[string]s
d := det.(*model.Detection)
_, exists := enabledDetections[d.PublicID]
if exists {
return nil, errors.New(fmt.Sprintf("duplicate detection with public ID %s", d.PublicID))
return nil, fmt.Errorf("duplicate detection with public ID %s", d.PublicID)
}
enabledDetections[d.PublicID] = d
}
Expand Down
Loading
Loading