Skip to content

Commit

Permalink
Merge pull request #425 from Security-Onion-Solutions/cogburn/git-errors
Browse files Browse the repository at this point in the history 
Refactored Git Work
  • Loading branch information
@coreyogburn
coreyogburn authored Apr 11, 2024
2 parents d7b1f0f + 387c283 commit 557618a
Show file tree
Hide file tree
Showing 3 changed files with 171 additions and 188 deletions.
105 changes: 13 additions & 92 deletions server/modules/elastalert/elastalert.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,15 @@ import (
"io"
"io/fs"
"net/http"
"net/url"
"os"
"os/exec"
"path"
"path/filepath"
"reflect"
"regexp"
"strings"
"sync"
"time"

"github.com/go-git/go-git/v5"
"github.com/samber/lo"
"github.com/security-onion-solutions/securityonion-soc/model"
"github.com/security-onion-solutions/securityonion-soc/module"
Expand Down Expand Up @@ -416,7 +413,7 @@ func (e *ElastAlertEngine) startCommunityRuleImport() {
templateFound = true
}

var allRepos map[string]*module.RuleRepo
allRepos := map[string]*module.RuleRepo{}
var repoChanges bool

var zips map[string][]byte
Expand All @@ -436,9 +433,15 @@ func (e *ElastAlertEngine) startCommunityRuleImport() {
continue
}

allRepos, repoChanges, err = e.updateRepos()
var dirtyRepos map[string]*mutil.DirtyRepo

dirtyRepos, repoChanges, err = mutil.UpdateRepos(&e.isRunning, e.reposFolder, e.rulesRepos)
if err != nil {
log.WithError(err).Error("unable to update sigma repos")
if strings.Contains(err.Error(), "module stopped") {
break
}

log.WithError(err).Error("unable to update Sigma repos")

if e.notify {
e.srv.Host.Broadcast("detection-sync", "detection", server.SyncStatus{
Expand All @@ -449,6 +452,10 @@ func (e *ElastAlertEngine) startCommunityRuleImport() {

continue
}

for k, v := range dirtyRepos {
allRepos[k] = v.Repo
}
} else {
// Possible airgap installation, or admin has disabled auto-updates.

Expand Down Expand Up @@ -585,92 +592,6 @@ func (e *ElastAlertEngine) startCommunityRuleImport() {
}
}

func (e *ElastAlertEngine) updateRepos() (allRepos map[string]*module.RuleRepo, anythingNew bool, err error) {
allRepos = map[string]*module.RuleRepo{} // map[repoPath]repo

// read existing repos
entries, err := os.ReadDir(e.reposFolder)
if err != nil {
log.WithError(err).Error("Failed to read sigma repos folder")
return nil, false, err
}

existingRepos := map[string]struct{}{}

for _, entry := range entries {
if !entry.IsDir() {
continue
}

existingRepos[entry.Name()] = struct{}{}
}

// pull or clone repos
for _, repo := range e.rulesRepos {
if !e.isRunning {
return nil, false, errModuleStopped
}

parser, err := url.Parse(repo.Repo)
if err != nil {
log.WithError(err).WithField("repo", repo).Error("Failed to parse repo URL, doing nothing with it")
continue
}

_, lastFolder := path.Split(parser.Path)
repoPath := filepath.Join(e.reposFolder, lastFolder)

allRepos[repoPath] = repo

if _, ok := existingRepos[lastFolder]; ok {
// repo already exists, pull
gitrepo, err := git.PlainOpen(repoPath)
if err != nil {
log.WithError(err).WithField("repo", gitrepo).Error("Failed to open repo, doing nothing with it")
continue
}

work, err := gitrepo.Worktree()
if err != nil {
log.WithError(err).WithField("repo", gitrepo).Error("Failed to get worktree, doing nothing with it")
continue
}

ctx, cancel := context.WithTimeout(e.srv.Context, time.Minute*5)

err = work.PullContext(ctx, &git.PullOptions{
Depth: 1,
SingleBranch: true,
})
if err != nil && err != git.NoErrAlreadyUpToDate {
cancel()
log.WithError(err).WithField("repo", repo).Error("Failed to pull repo, doing nothing with it")
continue
}
cancel()

if err == nil {
anythingNew = true
}
} else {
// repo does not exist, clone
_, err = git.PlainClone(repoPath, false, &git.CloneOptions{
Depth: 1,
SingleBranch: true,
URL: repo.Repo,
})
if err != nil {
log.WithError(err).WithField("repo", repo).Error("Failed to clone repo, doing nothing with it")
continue
}

anythingNew = true
}
}

return allRepos, anythingNew, nil
}

func (e *ElastAlertEngine) parseZipRules(pkgZips map[string][]byte) (detections []*model.Detection, errMap map[string]error) {
errMap = map[string]error{} // map[pkgName|fileName]error
defer func() {
Expand Down
115 changes: 19 additions & 96 deletions server/modules/strelka/strelka.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,8 @@ import (
"fmt"
"io/fs"
"net/http"
"net/url"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"strings"
Expand All @@ -29,7 +27,6 @@ import (
"github.com/security-onion-solutions/securityonion-soc/util"

"github.com/apex/log"
"github.com/go-git/go-git/v5"
)

var errModuleStopped = fmt.Errorf("strelka module has stopped running")
Expand Down Expand Up @@ -267,92 +264,34 @@ func (e *StrelkaEngine) startCommunityRuleImport() {
templateFound = true
}

// read existing repos
entries, err := os.ReadDir(e.reposFolder)
if err != nil {
log.WithError(err).Error("Failed to read yara repos folder")

if e.notify {
e.srv.Host.Broadcast("detection-sync", "detection", server.SyncStatus{
Engine: model.EngineNameStrelka,
Status: "error",
})
}

continue
}

existingRepos := map[string]struct{}{}

for _, entry := range entries {
if !entry.IsDir() {
continue
}

existingRepos[entry.Name()] = struct{}{}
}

upToDate := map[string]*module.RuleRepo{}

if e.autoUpdateEnabled {
// pull or clone repos
for _, repo := range e.rulesRepos {
if !e.isRunning {
allRepos, anythingNew, err := mutil.UpdateRepos(&e.isRunning, e.reposFolder, e.rulesRepos)
if err != nil {
if strings.Contains(err.Error(), "module stopped") {
break
}
}
if !anythingNew && !forceSync {
// no updates, skip
log.Info("Strelka sync found no changes")

parser, err := url.Parse(repo.Repo)
if err != nil {
log.WithError(err).WithField("repo", repo).Error("Failed to parse repo URL, doing nothing with it")
continue
}

_, lastFolder := path.Split(parser.Path)
repoPath := filepath.Join(e.reposFolder, lastFolder)

if _, ok := existingRepos[lastFolder]; ok {
// repo already exists, pull
gitrepo, err := git.PlainOpen(repoPath)
if err != nil {
log.WithError(err).WithField("repo", gitrepo).Error("Failed to open repo, doing nothing with it")
continue
}

work, err := gitrepo.Worktree()
if err != nil {
log.WithError(err).WithField("repo", gitrepo).Error("Failed to get worktree, doing nothing with it")
continue
}

ctx, cancel := context.WithTimeout(e.srv.Context, time.Minute*5)
mutil.WriteStateFile(e.IOManager, e.stateFilePath)

err = work.PullContext(ctx, &git.PullOptions{
Depth: 1,
SingleBranch: true,
if e.notify {
e.srv.Host.Broadcast("detection-sync", "detection", server.SyncStatus{
Engine: model.EngineNameStrelka,
Status: "success",
})
if err != nil && err != git.NoErrAlreadyUpToDate {
cancel()
log.WithError(err).WithField("repo", repo).Error("Failed to pull repo, doing nothing with it")
continue
}
cancel()
}

if err == nil || forceSync {
upToDate[repoPath] = repo
}
} else {
// repo does not exist, clone
_, err = git.PlainClone(repoPath, false, &git.CloneOptions{
Depth: 1,
SingleBranch: true,
URL: repo.Repo,
})
if err != nil {
log.WithError(err).WithField("repo", repo).Error("Failed to clone repo, doing nothing with it")
continue
}
continue
}

upToDate[repoPath] = repo
for k, v := range allRepos {
if v.WasModified {
upToDate[k] = v.Repo
}
}
} else {
Expand All @@ -364,22 +303,6 @@ func (e *StrelkaEngine) startCommunityRuleImport() {
return
}

if len(upToDate) == 0 {
// no updates, skip
log.Info("Strelka sync found no changes")

mutil.WriteStateFile(e.IOManager, e.stateFilePath)

if e.notify {
e.srv.Host.Broadcast("detection-sync", "detection", server.SyncStatus{
Engine: model.EngineNameStrelka,
Status: "success",
})
}

continue
}

communityDetections, err := e.srv.Detectionstore.GetAllCommunitySIDs(e.srv.Context, util.Ptr(model.EngineNameStrelka))
if err != nil {
log.WithError(err).Error("Failed to get all community SIDs")
Expand Down Expand Up @@ -726,7 +649,7 @@ func (e *StrelkaEngine) syncDetections(ctx context.Context) (errMap map[string]s
d := det.(*model.Detection)
_, exists := enabledDetections[d.PublicID]
if exists {
return nil, errors.New(fmt.Sprintf("duplicate detection with public ID %s", d.PublicID))
return nil, fmt.Errorf("duplicate detection with public ID %s", d.PublicID)
}
enabledDetections[d.PublicID] = d
}
Expand Down
Loading

0 comments on commit 557618a

Please sign in to comment.