Optimizations galore, and added file share analysis

.golangci.yml

@@ -0,0 +1,4 @@
+linters-settings:
+  govet:
+    check-shadowing: true
+    enable-all: true

modules/analyze/cli.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"os/exec"
 	"runtime"
+	"runtime/debug"
+	"time"
 
 	"github.com/lkarlslund/adalanche/modules/cli"
 	"github.com/lkarlslund/adalanche/modules/engine"
@@ -31,13 +33,18 @@ func init() {
 }
 
 func Execute(cmd *cobra.Command, args []string) error {
+	starttime := time.Now()
+
 	datapath := cmd.InheritedFlags().Lookup("datapath").Value.String()
 
 	objs, err := engine.Run(datapath)
 	if err != nil {
 		return err
 	}
 
+	// After all this loading and merging, it's time to do release unused RAM
+	debug.FreeOSMemory()
+
 	/*
 		switch command {
 		case "schemagraph":
@@ -156,6 +163,8 @@ func Execute(cmd *cobra.Command, args []string) error {
 		}
 	*/
 
+	log.Info().Msgf("Processing done in %v", time.Since(starttime))
+
 	err = WebService.Start(*bind, objs, *localhtml)
 	if err != nil {
 		return err

modules/analyze/export-graph.go

@@ -55,8 +55,8 @@ type CytoGraphData struct {
 type CytoElements []CytoFlatElement
 
 type CytoFlatElement struct {
-	Group string   `json:"group"` // nodes or edges
 	Data  CytoData `json:"data"`
+	Group string   `json:"group"` // nodes or edges
 }
 
 func GenerateCytoscapeJS(pg engine.PwnGraph, alldetails bool) (CytoGraph, error) {

modules/engine/attributes.go

@@ -32,6 +32,9 @@ const (
 	AttributeTypeFloat
 	AttributeTypeBool
 	AttributeTypeTime
+	AttributeTypeSID
+	AttributeTypeGUID
+	AttributeTypeBlob
 )
 
 type mergeapproverinfo struct {

modules/engine/attributevalue.go

@@ -16,6 +16,11 @@ func CompareAttributeValues(a, b AttributeValue) bool {
 	araw := a.Raw()
 	braw := b.Raw()
 	switch na := araw.(type) {
+	case bool:
+		nb, btype := braw.(bool)
+		if btype {
+			return na == nb
+		}
 	case string:
 		nb, btype := braw.(string)
 		if btype {
@@ -38,13 +43,20 @@ func CompareAttributeValues(a, b AttributeValue) bool {
 		}
 	case windowssecurity.SID:
 		nb, btype := braw.(windowssecurity.SID)
+		if btype {
+			return string(na) == string(nb)
+		}
+	case uuid.UUID:
+		nb, btype := braw.(uuid.UUID)
 		if btype {
 			return na == nb
 		}
+	default:
+		// Fallback
+		return a.String() == b.String()
 	}
 
-	// Fallback
-	return a.String() == b.String()
+	return false
 }
 
 type AttributeAndValues struct {
@@ -122,6 +134,7 @@ func (avo AttributeValueOne) StringSlice() []string {
 type AttributeValue interface {
 	String() string
 	Raw() interface{}
+	IsZero() bool
 }
 
 type AttributeValueObject struct {
@@ -136,6 +149,13 @@ func (avo AttributeValueObject) Raw() interface{} {
 	return (*Object)(avo.Object)
 }
 
+func (avo AttributeValueObject) IsZero() bool {
+	if avo.Object == nil {
+		return true
+	}
+	return len(avo.values) == 0
+}
+
 type AttributeValueString string
 
 func (as AttributeValueString) String() string {
@@ -146,6 +166,10 @@ func (as AttributeValueString) Raw() interface{} {
 	return string(as)
 }
 
+func (as AttributeValueString) IsZero() bool {
+	return len(as) == 0
+}
+
 type AttributeValueBlob []byte
 
 func (ab AttributeValueBlob) String() string {
@@ -156,6 +180,10 @@ func (ab AttributeValueBlob) Raw() interface{} {
 	return []byte(ab)
 }
 
+func (ab AttributeValueBlob) IsZero() bool {
+	return len(ab) == 0
+}
+
 type AttributeValueBool bool
 
 func (ab AttributeValueBool) String() string {
@@ -169,6 +197,10 @@ func (ab AttributeValueBool) Raw() interface{} {
 	return bool(ab)
 }
 
+func (ab AttributeValueBool) IsZero() bool {
+	return !bool(ab)
+}
+
 type AttributeValueInt int64
 
 func (as AttributeValueInt) String() string {
@@ -179,6 +211,10 @@ func (as AttributeValueInt) Raw() interface{} {
 	return int64(as)
 }
 
+func (as AttributeValueInt) IsZero() bool {
+	return int64(as) == 0
+}
+
 type AttributeValueTime time.Time
 
 func (as AttributeValueTime) String() string {
@@ -189,6 +225,10 @@ func (as AttributeValueTime) Raw() interface{} {
 	return time.Time(as)
 }
 
+func (as AttributeValueTime) IsZero() bool {
+	return time.Time(as).IsZero()
+}
+
 type AttributeValueSID windowssecurity.SID
 
 func (as AttributeValueSID) String() string {
@@ -199,6 +239,10 @@ func (as AttributeValueSID) Raw() interface{} {
 	return windowssecurity.SID(as)
 }
 
+func (as AttributeValueSID) IsZero() bool {
+	return windowssecurity.SID(as).IsNull()
+}
+
 type AttributeValueGUID uuid.UUID
 
 func (as AttributeValueGUID) String() string {
@@ -209,15 +253,19 @@ func (as AttributeValueGUID) Raw() interface{} {
 	return uuid.UUID(as)
 }
 
-type AttributeValueFiletime []byte
-
-func (as AttributeValueFiletime) String() string {
-	return string(as)
+func (as AttributeValueGUID) IsZero() bool {
+	return uuid.UUID(as).IsNil()
 }
 
-func (as AttributeValueFiletime) Raw() interface{} {
-	return string(as)
-}
+// type AttributeValueFiletime []byte
+
+// func (as AttributeValueFiletime) String() string {
+// 	return string(as)
+// }
+
+// func (as AttributeValueFiletime) Raw() interface{} {
+// 	return string(as)
+// }
 
 // func (as AttributeValueFiletime) AsTime() time.Time {
 // 	return nil

modules/engine/loaders.go

@@ -32,20 +32,22 @@ type Loader interface {
 var (
 	ErrUninterested = errors.New("plugin is not interested in this file, try harder")
 
-	loaders []Loader
+	loadergenerators []LoaderGenerator
 )
 
-func AddLoader(loader Loader) LoaderID {
-	loaders = append(loaders, loader)
-	return LoaderID(len(loaders) - 1)
+type LoaderGenerator func() Loader
+
+func AddLoader(lg LoaderGenerator) LoaderID {
+	loadergenerators = append(loadergenerators, lg)
+	return LoaderID(len(loadergenerators) - 1)
 }
 
 func NewLoaderObjects(ld Loader) *Objects {
 	aos := NewObjects()
 	aos.AddDefaultFlex(MetaDataSource, AttributeValueString(ld.Name()))
 
 	// Add the root node
-	rootnode := NewObject(Name, AttributeValueString(ld.Name()))
+	rootnode := NewObject(Name, ld.Name())
 	aos.Add(rootnode)
 	aos.SetRoot(rootnode)
 
@@ -58,19 +60,11 @@ type loaderobjects struct {
 }
 
 // Load runs all registered loaders
-func Load(path string, cb ProgressCallbackFunc) ([]loaderobjects, error) {
+func Load(loaders []Loader, path string, cb ProgressCallbackFunc) ([]loaderobjects, error) {
 	if st, err := os.Stat(path); err != nil || !st.IsDir() {
 		return nil, fmt.Errorf("%v is no a directory", path)
 	}
 
-	for _, loader := range loaders {
-		log.Debug().Msgf("Initializing loader for %v", loader.Name())
-		err := loader.Init()
-		if err != nil {
-			return nil, err
-		}
-	}
-
 	log.Info().Msgf("Scanning for data files from %v ...", path)
 	type fs struct {
 		filename string
@@ -100,7 +94,7 @@ func Load(path string, cb ProgressCallbackFunc) ([]loaderobjects, error) {
 
 	var skipped int
 	for _, file := range files {
-		var fileerr error
+		fileerr := ErrUninterested
 	loaderloop:
 		for _, loader := range loaders {
 			fileerr = loader.Load(file.filename, cb)