Skip to content

Commit

Permalink
rules: improved operator list parsing and conversion
Browse files Browse the repository at this point in the history 
Previously when creating a new rule we followed these steps:
 - Create a new protobuf Rule object from the ruleseditor or the
   pop-ups.
 - If the rule contained more than one operator, we converted the
   list of operators to a JSON string.
 - This JSON string was sent back to the daemon, and saved to the
   DB.
 - The list of operators were never expanded on the GUI, i.e., they
   were not saved as a list of protobuf Operator objects.
 - Once received in the daemon, the JSON string was parsed and
   converted to a protobuf Operator list of objects.
   Both, the JSON string and the list of protobuf Operator objects were
   saved to disk, but the JSON string was ignored when loading the
   rules.

Saving the list of operators as a JSON string was a problem if you
wanted to create or modify rules without the GUI.

Now when creating or modifying rules from the GUI, the list of operators
is no longer converted to JSON string. Instead the list is sent to the
daemon as a list of protobuf Operators, and saved as JSON objects.

Notes:
 - The JSON string is no longer saved to disk as part of the rules.
 - The list of operators is still saved as JSON string to the DB.
 - About not enabled rules:
   Previously, not enabled rules only had the list of operators as JSON
   string, with the field list:[] empty.
   Now the list of operators is saved as JSON objects, but if the rule
   is not enabled, it won't be parsed/loaded.

Closes #1047

(cherry picked from commit b930510)
  • Loading branch information
@gustavo-iniguez-goya
gustavo-iniguez-goya committed Jun 21, 2024
1 parent 4e7f1ae commit fe66f9a
Show file tree
Hide file tree
Showing 12 changed files with 382 additions and 122 deletions.
221 changes: 118 additions & 103 deletions daemon/rule/loader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,70 @@ func (l *Loader) Load(path string) error {
return nil
}

// Add adds a rule to the list of rules, and optionally saves it to disk.
func (l *Loader) Add(rule *Rule, saveToDisk bool) error {
l.addUserRule(rule)
if saveToDisk {
fileName := filepath.Join(l.path, fmt.Sprintf("%s.json", rule.Name))
return l.Save(rule, fileName)
}
return nil
}

// Replace adds a rule to the list of rules, and optionally saves it to disk.
func (l *Loader) Replace(rule *Rule, saveToDisk bool) error {
if err := l.replaceUserRule(rule); err != nil {
return err
}
if saveToDisk {
l.Lock()
defer l.Unlock()

fileName := filepath.Join(l.path, fmt.Sprintf("%s.json", rule.Name))
return l.Save(rule, fileName)
}
return nil
}

// Save a rule to disk.
func (l *Loader) Save(rule *Rule, path string) error {
rule.Updated = time.Now().Format(time.RFC3339)
raw, err := json.MarshalIndent(rule, "", " ")
if err != nil {
return fmt.Errorf("Error while saving rule %s to %s: %s", rule, path, err)
}

if err = ioutil.WriteFile(path, raw, 0600); err != nil {
return fmt.Errorf("Error while saving rule %s to %s: %s", rule, path, err)
}

return nil
}

// Delete deletes a rule from the list by name.
// If the duration is Always (i.e: saved on disk), it'll attempt to delete
// it from disk.
func (l *Loader) Delete(ruleName string) error {
l.Lock()
defer l.Unlock()

rule := l.rules[ruleName]
if rule == nil {
return nil
}
l.cleanListsRule(rule)

delete(l.rules, ruleName)
l.sortRules()

if rule.Duration != Always {
return nil
}

log.Info("Delete() rule: %s", rule)
return l.deleteRuleFromDisk(ruleName)
}

func (l *Loader) loadRule(fileName string) error {
raw, err := ioutil.ReadFile(fileName)
if err != nil {
Expand All @@ -117,7 +181,13 @@ func (l *Loader) loadRule(fileName string) error {
l.cleanListsRule(oldRule)
}

if r.Enabled {
if !r.Enabled {
// XXX: we only parse and load the Data field if the rule is disabled and the Data field is not empty
// the rule will remain disabled.
if err = l.unmarshalOperatorList(&r.Operator); err != nil {
return err
}
} else {
if err := r.Operator.Compile(); err != nil {
log.Warning("Operator.Compile() error: %s: %s", err, r.Operator.Data)
return fmt.Errorf("(1) Error compiling rule: %s", err)
Expand Down Expand Up @@ -191,41 +261,6 @@ func (l *Loader) cleanListsRule(oldRule *Rule) {
}
}

func (l *Loader) liveReloadWorker() {
l.liveReloadRunning = true

log.Debug("Rules watcher started on path %s ...", l.path)
if err := l.watcher.Add(l.path); err != nil {
log.Error("Could not watch path: %s", err)
l.liveReloadRunning = false
return
}

for {
select {
case event := <-l.watcher.Events:
// a new rule json file has been created or updated
if event.Op&fsnotify.Write == fsnotify.Write {
if strings.HasSuffix(event.Name, ".json") {
log.Important("Ruleset changed due to %s, reloading ...", path.Base(event.Name))
if err := l.loadRule(event.Name); err != nil {
log.Warning("%s", err)
}
}
} else if event.Op&fsnotify.Remove == fsnotify.Remove {
if strings.HasSuffix(event.Name, ".json") {
log.Important("Rule deleted %s", path.Base(event.Name))
// we only need to delete from memory rules of type Always,
// because the Remove event is of a file, i.e.: Duration == Always
l.deleteRule(event.Name)
}
}
case err := <-l.watcher.Errors:
log.Error("File system watcher error: %s", err)
}
}
}

func (l *Loader) isTemporary(r *Rule) bool {
return r.Duration != Restart && r.Duration != Always && r.Duration != Once
}
Expand All @@ -247,6 +282,18 @@ func (l *Loader) setUniqueName(rule *Rule) {
}
}

// Deprecated: rule.Operator.Data no longer holds the operator list in json format as string.
func (l *Loader) unmarshalOperatorList(op *Operator) error {
if op.Type == List && len(op.List) == 0 && op.Data != "" {
if err := json.Unmarshal([]byte(op.Data), &op.List); err != nil {
return fmt.Errorf("error loading rule of type list: %s", err)
}
op.Data = ""
}

return nil
}

func (l *Loader) sortRules() {
l.rulesKeys = make([]string, 0, len(l.rules))
for k := range l.rules {
Expand Down Expand Up @@ -278,22 +325,21 @@ func (l *Loader) replaceUserRule(rule *Rule) (err error) {
l.cleanListsRule(oldRule)
}

if err := l.unmarshalOperatorList(&rule.Operator); err != nil {
log.Error(err.Error())
}

if rule.Enabled {
if err := rule.Operator.Compile(); err != nil {
log.Warning("Operator.Compile() error: %s: %s", err, rule.Operator.Data)
return fmt.Errorf("(2) Error compiling rule: %s", err)
return fmt.Errorf("(2) error compiling rule: %s", err)
}

if rule.Operator.Type == List {
// TODO: use List protobuf object instead of un/marshalling to/from json
if err = json.Unmarshal([]byte(rule.Operator.Data), &rule.Operator.List); err != nil {
return fmt.Errorf("Error loading rule of type list: %s", err)
}

for i := 0; i < len(rule.Operator.List); i++ {
if err := rule.Operator.List[i].Compile(); err != nil {
log.Warning("Operator.Compile() error: %s: ", err)
return fmt.Errorf("(2) Error compiling list rule: %s", err)
return fmt.Errorf("(2) error compiling list rule: %s", err)
}
}
}
Expand Down Expand Up @@ -333,70 +379,39 @@ func (l *Loader) scheduleTemporaryRule(rule Rule) error {
return nil
}

// Add adds a rule to the list of rules, and optionally saves it to disk.
func (l *Loader) Add(rule *Rule, saveToDisk bool) error {
l.addUserRule(rule)
if saveToDisk {
fileName := filepath.Join(l.path, fmt.Sprintf("%s.json", rule.Name))
return l.Save(rule, fileName)
}
return nil
}

// Replace adds a rule to the list of rules, and optionally saves it to disk.
func (l *Loader) Replace(rule *Rule, saveToDisk bool) error {
if err := l.replaceUserRule(rule); err != nil {
return err
}
if saveToDisk {
l.Lock()
defer l.Unlock()

fileName := filepath.Join(l.path, fmt.Sprintf("%s.json", rule.Name))
return l.Save(rule, fileName)
}
return nil
}

// Save a rule to disk.
func (l *Loader) Save(rule *Rule, path string) error {
// When saving the rule, use always RFC3339 format for the Created field (#1140).
rule.Updated = time.Now().Format(time.RFC3339)

raw, err := json.MarshalIndent(rule, "", " ")
if err != nil {
return fmt.Errorf("Error while saving rule %s to %s: %s", rule, path, err)
}

if err = ioutil.WriteFile(path, raw, 0600); err != nil {
return fmt.Errorf("Error while saving rule %s to %s: %s", rule, path, err)
}

return nil
}

// Delete deletes a rule from the list by name.
// If the duration is Always (i.e: saved on disk), it'll attempt to delete
// it from disk.
func (l *Loader) Delete(ruleName string) error {
l.Lock()
defer l.Unlock()
func (l *Loader) liveReloadWorker() {
l.liveReloadRunning = true

rule := l.rules[ruleName]
if rule == nil {
return nil
log.Debug("Rules watcher started on path %s ...", l.path)
if err := l.watcher.Add(l.path); err != nil {
log.Error("Could not watch path: %s", err)
l.liveReloadRunning = false
return
}
l.cleanListsRule(rule)

delete(l.rules, ruleName)
l.sortRules()

if rule.Duration != Always {
return nil
for {
select {
case event := <-l.watcher.Events:
// a new rule json file has been created or updated
if event.Op&fsnotify.Write == fsnotify.Write {
if strings.HasSuffix(event.Name, ".json") {
log.Important("Ruleset changed due to %s, reloading ...", path.Base(event.Name))
if err := l.loadRule(event.Name); err != nil {
log.Warning("%s", err)
}
}
} else if event.Op&fsnotify.Remove == fsnotify.Remove {
if strings.HasSuffix(event.Name, ".json") {
log.Important("Rule deleted %s", path.Base(event.Name))
// we only need to delete from memory rules of type Always,
// because the Remove event is of a file, i.e.: Duration == Always
l.deleteRule(event.Name)
}
}
case err := <-l.watcher.Errors:
log.Error("File system watcher error: %s", err)
}
}

log.Info("Delete() rule: %s", rule)
return l.deleteRuleFromDisk(ruleName)
}

// FindFirstMatch will try match the connection against the existing rule set.
Expand Down
52 changes: 52 additions & 0 deletions daemon/rule/loader_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package rule

import (
"fmt"
"io"
"math/rand"
"os"
Expand Down Expand Up @@ -95,6 +96,57 @@ func TestRuleLoaderInvalidRegexp(t *testing.T) {
})
}

// Test rules of type operator.list. There're these scenarios:
// - Enabled rules:
// * operator Data field is ignored if it contains the list of operators as json string.
// * the operarots list is expanded as json objecs under "list": []
// For new rules (> v1.6.3), Data field will be empty.
//
// - Disabled rules
// * (old) the Data field contains the list of operators as json string, and the list of operarots is empty.
// * Data field empty, and the list of operators expanded.
// In all cases the list of operators must be loaded.
func TestRuleLoaderList(t *testing.T) {
l, err := NewLoader(true)
if err != nil {
t.Fail()
}

testRules := map[string]string{
"rule-with-operator-list": "testdata/rule-operator-list.json",
"rule-disabled-with-operators-list-as-json-string": "testdata/rule-disabled-operator-list.json",
"rule-disabled-with-operators-list-expanded": "testdata/rule-disabled-operator-list-expanded.json",
"rule-with-operator-list-data-empty": "testdata/rule-operator-list-data-empty.json",
}

for name, path := range testRules {
t.Run(fmt.Sprint("loadRule() ", path), func(t *testing.T) {
if err := l.loadRule(path); err != nil {
t.Error(fmt.Sprint("loadRule() ", path, " error:"), err)
}
t.Log("Test: List rule:", name, path)
r, found := l.rules[name]
if !found {
t.Error(fmt.Sprint("loadRule() ", path, " not in the list:"), l.rules)
}
// Starting from > v1.6.3, after loading a rule of type List, the field Operator.Data is emptied, if the Data contained the list of operators as json.
if len(r.Operator.List) != 2 {
t.Error(fmt.Sprint("loadRule() ", path, " operator List not loaded:"), r)
}
if r.Operator.List[0].Type != Simple ||
r.Operator.List[0].Operand != OpProcessPath ||
r.Operator.List[0].Data != "/usr/bin/telnet" {
t.Error(fmt.Sprint("loadRule() ", path, " operator List 0 not loaded:"), r)
}
if r.Operator.List[1].Type != Simple ||
r.Operator.List[1].Operand != OpDstPort ||
r.Operator.List[1].Data != "53" {
t.Error(fmt.Sprint("loadRule() ", path, " operator List 1 not loaded:"), r)
}
})
}
}

func TestLiveReload(t *testing.T) {
t.Parallel()
t.Log("Test rules loader with live reload")
Expand Down

0 comments on commit fe66f9a

Please sign in to comment.