additional logging; auto-enable yara rules

Dockerfile

@@ -22,9 +22,14 @@ RUN if [ "$VERSION" != "0.0.0" ]; then mkdir gitdocs && cd gitdocs && \
 RUN npm install jest jest-environment-jsdom --global
 RUN ./build.sh "$VERSION"
 
-RUN pip3 install sigma-cli pysigma-backend-elasticsearch pysigma-pipeline-windows yara-python --break-system-packages
+RUN pip3 install sigma-cli pysigma-backend-elasticsearch pysigma-pipeline-windows --break-system-packages
 RUN sed -i 's/#!\/usr\/bin\/python3/#!\/usr\/bin\/env python/g' /usr/bin/sigma
 
+# Build specific version of yara-python - needs to be pinned to Strelka's version.
+FROM ghcr.io/security-onion-solutions/python:3-slim as stage_2
+RUN apt-get update && apt-get install -y gcc python3-dev libssl-dev
+RUN pip3 install yara-python==4.3.1
+
 FROM ghcr.io/security-onion-solutions/python:3-slim
 
 ARG UID=939
@@ -51,6 +56,8 @@ COPY --from=builder /build/sensoroni.json .
 COPY --from=builder /build/gitdocs/_build/html ./html/docs
 COPY --from=builder /usr/lib/python3.11/site-packages /usr/local/lib/python3.9/site-packages
 COPY --from=builder /usr/bin/sigma /usr/bin/sigma
+COPY --from=stage_2 /usr/local/lib/python3.9/site-packages/yara_python-4.3.1.dist-info /usr/local/lib/python3.9/site-packages/
+COPY --from=stage_2 /usr/local/lib/python3.9/site-packages/yara.cpython-39-x86_64-linux-gnu.so /usr/local/lib/python3.9/site-packages/
 RUN find html/js -name "*test*.js" -delete
 RUN chmod u+x scripts/*
 RUN chown 939:939 scripts/*

server/modules/elastalert/elastalert.go

@@ -79,14 +79,18 @@ type ElastAlertEngine struct {
 
 func checkRulesetEnabled(e *ElastAlertEngine, det *model.Detection) {
 	det.IsEnabled = false
+	if det.Ruleset == nil || det.Severity == "" {
+		return
+	}
+
+	// Combine Ruleset and Severity into a single string
 	metaCombined := *det.Ruleset + "+" + string(det.Severity)
 	for _, rule := range e.autoEnabledSigmaRules {
-		if rule == metaCombined {
+		if strings.EqualFold(rule, metaCombined) {
 			det.IsEnabled = true
 			break
 		}
 	}
-
 }
 
 func NewElastAlertEngine(srv *server.Server) *ElastAlertEngine {

server/modules/elastalert/elastalert_test.go

@@ -44,8 +44,12 @@ func TestCheckAutoEnabledSigmaRule(t *testing.T) {
 		expected bool
 	}{
 		{"securityonion-resources rule with high severity, rule enabled", "securityonion-resources", model.SeverityHigh, true},
+		{"securityonion-resources rule with high severity upper case, rule enabled", "securityonion-RESOURCES", model.SeverityHigh, true},
 		{"core rule with critical severity, rule enabled", "core", model.SeverityCritical, true},
 		{"core rule with high severity, rule not enabled", "core", model.SeverityHigh, false},
+		{"empty ruleset, high severity, rule not enabled", "", model.SeverityHigh, false},
+		{"core ruleset, empty severity, rule not enabled", "core", "", false},
+		{"empty ruleset, empty severity, rule not enabled", "", "", false},
 	}
 
 	for _, tt := range tests {

server/modules/strelka/strelka.go

@@ -51,6 +51,7 @@ type StrelkaEngine struct {
 	communityRulesImportFrequencySeconds int
 	yaraRulesFolder                      string
 	reposFolder                          string
+	autoEnabledYaraRules                 []string
 	rulesRepos                           []*module.RuleRepo
 	compileYaraPythonScriptPath          string
 	allowRegex                           *regexp.Regexp
@@ -62,6 +63,21 @@ type StrelkaEngine struct {
 	IOManager
 }
 
+func checkRulesetEnabled(e *StrelkaEngine, det *model.Detection) {
+	det.IsEnabled = false
+
+	if det.Ruleset == nil {
+		return
+	}
+
+	for _, rule := range e.autoEnabledYaraRules {
+		if strings.EqualFold(rule, *det.Ruleset) {
+			det.IsEnabled = true
+			break
+		}
+	}
+}
+
 func NewStrelkaEngine(srv *server.Server) *StrelkaEngine {
 	return &StrelkaEngine{
 		srv:       srv,
@@ -83,6 +99,7 @@ func (e *StrelkaEngine) Init(config module.ModuleConfig) (err error) {
 	e.compileYaraPythonScriptPath = module.GetStringDefault(config, "compileYaraPythonScriptPath", "/opt/so/conf/strelka/compile_yara.py")
 	e.compileRules = module.GetBoolDefault(config, "compileRules", true)
 	e.autoUpdateEnabled = module.GetBoolDefault(config, "autoUpdateEnabled", false)
+	e.autoEnabledYaraRules = module.GetStringArrayDefault(config, "autoEnabledYaraRules", []string{"securityonion-yara"})
 
 	e.rulesRepos, err = module.GetReposDefault(config, "rulesRepos", []*module.RuleRepo{
 		{
@@ -290,7 +307,7 @@ func (e *StrelkaEngine) startCommunityRuleImport() {
 			}
 
 			for k, v := range allRepos {
-				if v.WasModified {
+				if v.WasModified || forceSync {
 					upToDate[k] = v.Repo
 				}
 			}
@@ -361,6 +378,10 @@ func (e *StrelkaEngine) startCommunityRuleImport() {
 
 				for _, rule := range parsed {
 					det := rule.ToDetection(repo.License, filepath.Base(repopath))
+					log.WithFields(log.Fields{
+						"rule.uuid": det.PublicID,
+						"rule.name": det.Title,
+					}).Info("Strelka community sync - processing YARA rule")
 
 					comRule, exists := communityDetections[det.PublicID]
 					if exists {
@@ -371,13 +392,22 @@ func (e *StrelkaEngine) startCommunityRuleImport() {
 
 					if exists {
 						// pre-existing detection, update it
+						log.WithFields(log.Fields{
+							"rule.uuid": det.PublicID,
+							"rule.name": det.Title,
+						}).Info("Updating Yara detection")
 						det, err = e.srv.Detectionstore.UpdateDetection(e.srv.Context, det)
 						if err != nil {
 							log.WithError(err).WithField("det", det).Error("Failed to update detection")
 							continue
 						}
 					} else {
 						// new detection, create it
+						log.WithFields(log.Fields{
+							"rule.uuid": det.PublicID,
+							"rule.name": det.Title,
+						}).Info("Creating new Yara detection")
+						checkRulesetEnabled(e, det)
 						det, err = e.srv.Detectionstore.CreateDetection(e.srv.Context, det)
 						if err != nil {
 							log.WithError(err).WithField("det", det).Error("Failed to create detection")
@@ -654,26 +684,28 @@ func (e *StrelkaEngine) syncDetections(ctx context.Context) (errMap map[string]s
 		enabledDetections[d.PublicID] = d
 	}
 
-	filename := filepath.Join(e.yaraRulesFolder, "enabled_rules.yar")
-
-	if len(enabledDetections) == 0 {
-		err = e.DeleteFile(filename)
-		if err != nil && !os.IsNotExist(err) {
-			return nil, err
-		}
-
-		return nil, nil
+	// Clear existing .yar files in the directory
+	files, err := e.ReadDir(e.yaraRulesFolder)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read directory: %v", err)
 	}
 
-	buf := bytes.Buffer{}
-
-	for _, det := range enabledDetections {
-		buf.WriteString(det.Content + "\n")
+	for _, file := range files {
+		if filepath.Ext(file.Name()) == ".yar" {
+			err := e.DeleteFile(filepath.Join(e.yaraRulesFolder, file.Name()))
+			if err != nil {
+				return nil, fmt.Errorf("failed to delete existing .yar file %s: %v", file.Name(), err)
+			}
+		}
 	}
 
-	err = e.WriteFile(filename, buf.Bytes(), 0644)
-	if err != nil {
-		return nil, err
+	// Process and write new .yar files
+	for publicId, det := range enabledDetections {
+		filename := filepath.Join(e.yaraRulesFolder, fmt.Sprintf("%s.yar", publicId))
+		err := e.WriteFile(filename, []byte(det.Content), 0644)
+		if err != nil {
+			return nil, fmt.Errorf("failed to write file for detection %s: %v", publicId, err)
+		}
 	}
 
 	if e.compileRules {

server/modules/strelka/strelka_test.go

@@ -205,6 +205,33 @@ func TestStrelkaModule(t *testing.T) {
 	assert.Same(t, mod, srv.DetectionEngines[model.EngineNameStrelka])
 }
 
+func TestCheckAutoEnabledYaraRule(t *testing.T) {
+	e := &StrelkaEngine{
+		autoEnabledYaraRules: []string{"securityonion-yara"},
+	}
+
+	tests := []struct {
+		name     string
+		ruleset  string
+		expected bool
+	}{
+		{"securityonion-yara rule, rule enabled", "securityonion-yara", true},
+		{"securityonion-YARA rule upper case, rule enabled", "securityonion-YARA", true},
+		{"securityonion-fake rule, rule not enabled", "securityonion-fake", false},
+		{"no ruleset, rule not enabled", "", false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			det := &model.Detection{
+				Ruleset: util.Ptr(tt.ruleset),
+			}
+			checkRulesetEnabled(e, det)
+			assert.Equal(t, tt.expected, det.IsEnabled)
+		})
+	}
+}
+
 func TestSyncStrelka(t *testing.T) {
 	table := []struct {
 		Name           string
@@ -230,7 +257,9 @@ func TestSyncStrelka(t *testing.T) {
 					},
 				}, nil)
 
-				mio.EXPECT().WriteFile(gomock.Any(), []byte(simpleRule+"\n"+simpleRule+"\n"), fs.FileMode(0644)).Return(nil)
+				mio.EXPECT().ReadDir("yaraRulesFolder").Return(nil, nil)
+
+				mio.EXPECT().WriteFile(gomock.Any(), []byte(simpleRule), fs.FileMode(0644)).Return(nil).MaxTimes(2)
 
 				mio.EXPECT().ExecCommand(gomock.Cond(func(c any) bool {
 					cmd := c.(*exec.Cmd)
@@ -247,13 +276,6 @@ func TestSyncStrelka(t *testing.T) {
 				})).Return([]byte{}, 0, time.Duration(0), nil)
 			},
 		},
-		{
-			Name: "No Enabled Rules",
-			InitMock: func(mockDetStore *servermock.MockDetectionstore, mio *mock.MockIOManager) {
-				mockDetStore.EXPECT().Query(gomock.Any(), gomock.Any(), gomock.Any()).Return([]interface{}{}, nil)
-				mio.EXPECT().DeleteFile("yaraRulesFolder/enabled_rules.yar").Return(nil)
-			},
-		},
 	}
 
 	ctx := context.Background()