Improved cross forest object joins, added icon for foreign security p…

…rincipals, merge validator functions implemented

modules/analyze/html/graph.js

@@ -274,6 +274,13 @@ var cytostyle = [{
         "background-color": "lightgreen"
     }
 },
+{
+    selector: 'node[type="ForeignSecurityPrincipal"]',
+    style: {
+        "background-image": "icons/badge_black_24dp.svg",
+        "background-color": "lightgreen"
+    }
+},
 {
     selector: 'node[type="Service"]',
     style: {

modules/engine/attributes.go

@@ -19,11 +19,11 @@ type attributeinfo struct {
 	multi     bool // If true, this attribute can have multiple values
 	nonunique bool // Doing a Find on this attribute will return multiple results
 	merge     bool // If true, objects can be merged on this attribute
-	mf        mergefunc
 	onset     AttributeSetFunc
 	onget     AttributeGetFunc
 }
 
+var mergeapprovers []mergefunc
 var attributenums []attributeinfo
 
 var (
@@ -52,17 +52,11 @@ var (
 	ObjectCategoryGUID = NewAttribute("objectCategoryGUID") // Used for caching the GUIDs
 
 	MetaDataSource = NewAttribute("_datasource").Multi()
-	UniqueSource   = NewAttribute("_source").Merge(func(attr Attribute, a, b *Object) (*Object, error) {
-		// Prevents objects from vastly different sources to join across them
-		if a.HasAttr(attr) && b.HasAttr(attr) && a.OneAttrString(attr) != b.OneAttrString(attr) {
-			return nil, ErrDontMerge
-		}
-		return nil, ErrMergeOnOtherAttr
-	})
+	UniqueSource   = NewAttribute("_source")
 
 	IPAddress          = NewAttribute("IPAddress")
-	Hostname           = NewAttribute("Hostname").Merge(nil)
-	DownLevelLogonName = NewAttribute("DownLevelLogonName").Merge(nil)
+	Hostname           = NewAttribute("Hostname").Merge()
+	DownLevelLogonName = NewAttribute("DownLevelLogonName").Merge()
 	NetbiosDomain      = NewAttribute("netbiosDomain") // Used to merge users with - if we only have a DOMAIN\USER type of info
 
 	MetaProtectedUser           = NewAttribute("_protecteduser")
@@ -87,6 +81,24 @@ var (
 	_ = NewAttribute("dSCorePropagationData")
 )
 
+func init() {
+	AddMergeApprover(func(a, b *Object) (*Object, error) {
+		// Prevents objects from vastly different sources to join across them
+		if a.HasAttr(UniqueSource) && b.HasAttr(UniqueSource) {
+			for _, v := range a.Attr(UniqueSource).Slice() {
+				if b.HasAttrValue(UniqueSource, v) {
+					// At least one of the UniqueSources overlap
+					return nil, nil
+				}
+			}
+			// Don't cross the streams, protonic reversal may never happen
+			return nil, ErrDontMerge
+		}
+
+		return nil, nil
+	})
+}
+
 type Attribute uint16
 
 var attributemutex sync.RWMutex
@@ -148,28 +160,25 @@ func (a Attribute) NonUnique() Attribute {
 }
 
 var ErrDontMerge = errors.New("Dont merge objects using any methods")
-var ErrMergeOnOtherAttr = errors.New("Merge on other attribute")
 var ErrMergeOnThis = errors.New("Merge on this attribute")
 
-type mergefunc func(attr Attribute, a, b *Object) (*Object, error)
+type mergefunc func(a, b *Object) (*Object, error)
 
 func StandardMerge(attr Attribute, a, b *Object) (*Object, error) {
 	return nil, nil
 }
 
-func (a Attribute) Merge(mf mergefunc) Attribute {
+func (a Attribute) Merge() Attribute {
 	ai := attributenums[a]
 	ai.merge = true
-	if mf != nil {
-		if ai.mf != nil {
-			log.Fatal().Msgf("Attribute %v already has a merge function", a)
-		}
-		ai.mf = mf
-	}
 	attributenums[a] = ai
 	return a
 }
 
+func AddMergeApprover(mf mergefunc) {
+	mergeapprovers = append(mergeapprovers, mf)
+}
+
 func (a Attribute) Tag(t string) Attribute {
 	ai := attributenums[a]
 	ai.tags = append(ai.tags, t)

modules/engine/object.go

@@ -79,13 +79,13 @@ type Object struct {
 	children  []*Object
 	members   []*Object
 	// objectclassguids   []uuid.UUID
-	memberof           []*Object
-	id                 uint32
-	guid               uuid.UUID
-	objectcategoryguid uuid.UUID
-	guidcached         bool
-	sidcached          bool
-	objecttype         ObjectType
+	memberof []*Object
+	id       uint32
+	guid     uuid.UUID
+	// objectcategoryguid uuid.UUID
+	guidcached bool
+	sidcached  bool
+	objecttype ObjectType
 }
 
 type Connection struct {
@@ -363,22 +363,14 @@ func (o *Object) Type() ObjectType {
 	return o.objecttype
 }
 
-// func (o *Object) ObjectClassGUIDs() []uuid.UUID {
-// 	if len(o.objectclassguids) == 0 {
-// 	}
-// 	return o.objectclassguids
-// }
-
 func (o *Object) ObjectCategoryGUID(ao *Objects) uuid.UUID {
-	if o.objectcategoryguid == NullGUID {
-		guid := o.OneAttrRaw(ObjectCategoryGUID)
-		if guid == nil {
-			o.objectcategoryguid = UnknownGUID
-		} else {
-			o.objectcategoryguid = guid.(uuid.UUID)
-		}
+	// if o.objectcategoryguid == NullGUID {
+	guid := o.OneAttrRaw(ObjectCategoryGUID)
+	if guid == nil {
+		return UnknownGUID
 	}
-	return o.objectcategoryguid
+	return guid.(uuid.UUID)
+	// return o.objectcategoryguid
 }
 
 func (o *Object) AttrString(attr Attribute) []string {
@@ -695,9 +687,11 @@ func (o *Object) set(a Attribute, values AttributeValues) {
 			panic("DownLevelLogon ends with \\")
 		}
 	}
+
 	if a == ObjectCategory || a == ObjectCategorySimple {
-		o.objectcategoryguid = NullGUID
+		o.objecttype = 0
 	}
+
 	if a == NTSecurityDescriptor {
 		for _, sd := range values.Slice() {
 			if err := o.cacheSecurityDescriptor([]byte(sd.Raw().(string))); err != nil {

modules/engine/objects.go

@@ -152,8 +152,8 @@ func (os *Objects) updateIndex(o *Object, warn bool) {
 				existing, dupe := os.uniqueindex[attribute][value.Raw()]
 				if dupe && existing != o {
 					log.Warn().Msgf("Duplicate index %v value %v when trying to add %v, already exists as %v, index still points to original object", attribute.String(), value.String(), o.Label(), existing.Label())
-					// log.Debug().Msgf("NEW: %v", o.String(os))
-					// log.Debug().Msgf("EXISTING: %v", existing.String(os))
+					log.Debug().Msgf("NEW: %v", o.StringNoACL())
+					log.Debug().Msgf("EXISTING: %v", existing.StringNoACL())
 					continue
 				}
 			}
@@ -233,38 +233,38 @@ func (os *Objects) Merge(attrtomerge []Attribute, o *Object) bool {
 
 func (os *Objects) merge(attrtomerge []Attribute, o *Object) bool {
 	if len(attrtomerge) > 0 {
-	outerattrloop:
 		for _, mergeattr := range attrtomerge {
 			for _, lookfor := range o.Attr(mergeattr).Slice() {
 				if lookfor == nil {
 					continue
 				}
-				if mergetarget, found := os.Find(mergeattr, lookfor); found {
-					// Let's merge
-					mf := attributenums[mergeattr].mf
-					if mf != nil {
-						res, err := mf(mergeattr, o, mergetarget)
-						switch err {
-						case ErrDontMerge:
-							return false
-						case ErrMergeOnOtherAttr:
-							continue outerattrloop
-						case ErrMergeOnThis, nil:
-							// Let the code below do the merge
-						default:
-							log.Warn().Msgf("Error merging %v: %v", o.Label(), err)
-							return false
-						}
-						if res != nil {
-							// Custom merge - how do we handle this?
-							log.Error().Msgf("Custom merge function not supported yet")
-							return false
+				if mergetargets, found := os.FindMulti(mergeattr, lookfor); found {
+
+				targetloop:
+					for _, mergetarget := range mergetargets {
+						for _, mf := range mergeapprovers {
+							if mf != nil {
+								res, err := mf(o, mergetarget)
+								switch err {
+								case ErrDontMerge:
+									continue targetloop
+								case ErrMergeOnThis, nil:
+									// Let the code below do the merge
+								default:
+									log.Fatal().Msgf("Error merging %v: %v", o.Label(), err)
+								}
+								if res != nil {
+									// Custom merge - how do we handle this?
+									log.Fatal().Msgf("Custom merge function not supported yet")
+									return false
+								}
+							}
 						}
+						log.Trace().Msgf("Merging %v with %v on attribute %v", o.Label(), mergetarget.Label(), mergeattr.String())
+						mergetarget.Absorb(o)
+						os.updateIndex(mergetarget, false)
+						return true
 					}
-					log.Trace().Msgf("Merging %v with %v on attribute %v", o.Label(), mergetarget.Label(), mergeattr.String())
-					mergetarget.Absorb(o)
-					os.updateIndex(mergetarget, false)
-					return true
 				}
 			}
 		}

modules/engine/processing.go

@@ -21,6 +21,9 @@ func Merge(aos []*Objects) (*Objects, error) {
 	// After merge we don't need all the indexes
 	globalobjects.DropIndexes()
 
+	// Let's not change anything in the original objects
+	globalobjects.DefaultValues = nil
+
 	globalroot := NewObject(
 		Name, AttributeValueString("adalanche root node"),
 		ObjectCategorySimple, AttributeValueString("Root"),

modules/integrations/activedirectory/analyze/adloader.go

@@ -96,7 +96,7 @@ func (ld *ADLoader) Init() error {
 					continue
 				}
 
-				thisao := ld.getShard(path)
+				thisao := ld.getShard(path, true)
 				err = ImportGPOInfo(ginfo, thisao)
 				if err != nil {
 					log.Warn().Msgf("Problem importing GPO: %v", err)
@@ -109,16 +109,23 @@ func (ld *ADLoader) Init() error {
 	return nil
 }
 
-func (ld *ADLoader) getShard(path string) *engine.Objects {
+func (ld *ADLoader) getShard(path string, gpo bool) *engine.Objects {
 	shard := filepath.Dir(path)
+
+	lookupshard := shard
+	if gpo {
+		// Load GPO stuff into their own shard
+		lookupshard += "_gpo"
+	}
+
 	var ao *engine.Objects
 	ld.importmutex.Lock()
-	ao = ld.dco[shard]
+	ao = ld.dco[lookupshard]
 	if ao == nil {
 		ao = engine.NewLoaderObjects(ld)
-
+		ao.AddDefaultFlex(engine.UniqueSource, engine.AttributeValueString(shard))
 		ao.SetThreadsafe(true)
-		ld.dco[shard] = ao
+		ld.dco[lookupshard] = ao
 	}
 	ld.importmutex.Unlock()
 	return ao
@@ -129,7 +136,7 @@ func (ld *ADLoader) Load(path string, cb engine.ProgressCallbackFunc) error {
 	case strings.HasSuffix(path, ".gpodata.json"):
 		ld.gpofiletoprocess <- path
 	case strings.HasSuffix(path, ".objects.msgp.lz4"):
-		ao := ld.getShard(path)
+		ao := ld.getShard(path, false)
 
 		cachefile, err := os.Open(path)
 		if err != nil {