diff --git a/modules/analyze/webservicefuncs.go b/modules/analyze/webservicefuncs.go index 2b8a8dc..1c44520 100644 --- a/modules/analyze/webservicefuncs.go +++ b/modules/analyze/webservicefuncs.go @@ -324,12 +324,14 @@ func analysisfuncs(ws *webservice) { // We dont support this yet, so merge all of them combinedmethods := edges_f.Merge(egdes_m).Merge(edges_l) - for _, source := range includeobjects.Slice() { - for _, target := range excludeobjects.Slice() { + includeobjects.Iterate(func(source *engine.Object) bool { + excludeobjects.Iterate(func(target *engine.Object) bool { newpg := engine.AnalyzePaths(source, target, ws.Objs, combinedmethods, engine.Probability(minprobability), maxdepth) pg.Merge(newpg) - } - } + return true + }) + return true + }) // pg = engine.AnalyzePaths(includeobjects.First(), excludeobjects.First(), ws.Objs, combinedmethods, engine.Probability(minprobability), 1) } else { opts := engine.NewAnalyzeObjectsOptions() @@ -657,11 +659,12 @@ func analysisfuncs(ws *webservice) { return includequery.Evaluate(o) }) - dns := make([]string, len(objects.Slice())) + dns := make([]string, 0, objects.Len()) - for i, o := range objects.Slice() { - dns[i] = o.DN() - } + objects.Iterate(func(o *engine.Object) bool { + dns = append(dns, o.DN()) + return true + }) err = encoder.Encode(dns) if err != nil { @@ -695,7 +698,7 @@ func analysisfuncs(ws *webservice) { return includequery.Evaluate(o) }) - err = encoder.Encode(objects.Slice()) + err = encoder.Encode(objects.AsSlice()) if err != nil { w.WriteHeader(400) // bad request w.Write([]byte(err.Error())) @@ -722,7 +725,7 @@ func analysisfuncs(ws *webservice) { HasLAPS bool `json:"haslaps,omitempty"` } var result []info - for _, object := range ws.Objs.Slice() { + ws.Objs.Iterate(func(object *engine.Object) bool { if object.Type() == engine.ObjectTypeUser && object.OneAttrString(engine.MetaWorkstation) != "1" && object.OneAttrString(engine.MetaServer) != "1" && @@ -772,7 +775,8 @@ func analysisfuncs(ws *webservice) { result = append(result, i) } - } + return true + }) data, err := json.MarshalIndent(result, "", " ") if err != nil { @@ -857,10 +861,11 @@ func analysisfuncs(ws *webservice) { } var pwnlinks int - for _, object := range ws.Objs.Slice() { + ws.Objs.Iterate(func(object *engine.Object) bool { pwnlinks += object.Edges(engine.Out).Len() - } - result.Statistics["Total"] = len(ws.Objs.Slice()) + return true + }) + result.Statistics["Total"] = ws.Objs.Len() result.Statistics["PwnConnections"] = pwnlinks data, _ := json.MarshalIndent(result, "", " ") diff --git a/modules/engine/analyzeobjects.go b/modules/engine/analyzeobjects.go index 428cbf0..800a8bc 100644 --- a/modules/engine/analyzeobjects.go +++ b/modules/engine/analyzeobjects.go @@ -74,11 +74,12 @@ func AnalyzeObjects(opts AnalyzeObjectsOptions) (pg Graph) { // Convert to our working map processinground := 1 - for _, object := range opts.IncludeObjects.Slice() { - implicatedobjectsmap[object] = &roundinfo{ + opts.IncludeObjects.Iterate(func(o *Object) bool { + implicatedobjectsmap[o] = &roundinfo{ roundadded: processinground, } - } + return true + }) // Methods and ObjectTypes allowed detectedges := opts.MethodsF diff --git a/modules/engine/attributes.go b/modules/engine/attributes.go index e549f3b..83a772e 100644 --- a/modules/engine/attributes.go +++ b/modules/engine/attributes.go @@ -103,6 +103,11 @@ var ( type Attribute uint16 +type AttributePair struct { + attribute1 Attribute + attribute2 Attribute +} + var attributemutex sync.RWMutex func NewAttribute(name string) Attribute { diff --git a/modules/engine/attributevalue.go b/modules/engine/attributevalue.go index 3d004a5..ef24857 100644 --- a/modules/engine/attributevalue.go +++ b/modules/engine/attributevalue.go @@ -148,6 +148,11 @@ type AttributeValue interface { IsZero() bool } +type AttributeValuePair struct { + Value1 AttributeValue + Value2 AttributeValue +} + type AttributeValueObject struct { *Object } diff --git a/modules/engine/index.go b/modules/engine/index.go new file mode 100644 index 0000000..35f07fe --- /dev/null +++ b/modules/engine/index.go @@ -0,0 +1,99 @@ +package engine + +import ( + "sync" +) + +type Index struct { + lookup map[AttributeValue][]*Object + sync.RWMutex +} + +func (i *Index) init() { + i.lookup = make(map[AttributeValue][]*Object) +} + +func (i *Index) Lookup(key AttributeValue) ([]*Object, bool) { + iv := AttributeValueToIndex(key) + i.RLock() + result, found := i.lookup[iv] + i.RUnlock() + return result, found +} + +func (i *Index) Add(key AttributeValue, o *Object, undupe bool) { + iv := AttributeValueToIndex(key) + i.Lock() + if undupe { + existing, _ := i.lookup[iv] + for _, dupe := range existing { + if dupe == o { + i.Unlock() + return + } + } + } + + existing, _ := i.lookup[iv] + i.lookup[iv] = append(existing, o) + i.Unlock() +} + +func (i *Index) Iterate(each func(key AttributeValue, objects []*Object) bool) { + i.RLock() + for key, value := range i.lookup { + if !each(key, value) { + break + } + } + i.RUnlock() +} + +type MultiIndex struct { + lookup map[AttributeValuePair][]*Object + sync.RWMutex +} + +func (i *MultiIndex) init() { + i.lookup = make(map[AttributeValuePair][]*Object) +} + +func (i *MultiIndex) Lookup(key, key2 AttributeValue) ([]*Object, bool) { + iv := AttributeValueToIndex(key) + iv2 := AttributeValueToIndex(key2) + + i.RLock() + result, found := i.lookup[AttributeValuePair{iv, iv2}] + i.RUnlock() + return result, found +} + +func (i *MultiIndex) Add(key, key2 AttributeValue, o *Object, undupe bool) { + iv := AttributeValueToIndex(key) + iv2 := AttributeValueToIndex(key2) + avp := AttributeValuePair{iv, iv2} + i.Lock() + if undupe { + existing, _ := i.lookup[avp] + for _, dupe := range existing { + if dupe == o { + i.Unlock() + return + } + } + } + + existing, _ := i.lookup[avp] + i.lookup[avp] = append(existing, o) + i.Unlock() +} + +func (i *MultiIndex) Iterate(each func(key, key2 AttributeValue, objects []*Object) bool) { + i.RLock() + for pair, value := range i.lookup { + if !each(pair.Value1, pair.Value2, value) { + break + } + } + i.RUnlock() +} diff --git a/modules/engine/objects.go b/modules/engine/objects.go index 04e7cc1..8d7e7dd 100644 --- a/modules/engine/objects.go +++ b/modules/engine/objects.go @@ -7,6 +7,7 @@ import ( "sync/atomic" "time" + gsync "github.com/SaveTheRbtz/generic-sync-map-go" "github.com/akyoto/cache" "github.com/gofrs/uuid" "github.com/lkarlslund/adalanche/modules/ui" @@ -23,68 +24,20 @@ type Objects struct { objectmutex sync.RWMutex - asarray []*Object // All objects in this collection, returned by .Slice() - asmap map[*Object]struct{} - indexlock sync.RWMutex - indexes []*Index - multiindexes map[uint32]*Index // Indexes for multiple attributes (lower attr < 16 || higher attr) - - idindex map[uint32]int // Offset into asarray + indexes []*Index // Uses atribute directly as slice offset for performance + multiindexes map[AttributePair]*MultiIndex // Uses a map for storage considerations + idindex gsync.MapOf[uint32, *Object] typecount typestatistics } -type Index struct { - lookup map[interface{}][]*Object - sync.RWMutex -} - -func (i *Index) Init() { - i.lookup = make(map[interface{}][]*Object) -} - -func (i *Index) Lookup(key interface{}) ([]*Object, bool) { - i.RLock() - results, found := i.lookup[key] - i.RUnlock() - return results, found -} - -func (i *Index) Add(key interface{}, o *Object, undupe bool) { - i.Lock() - if undupe { - existing := i.lookup[key] - for _, dupe := range existing { - if dupe == o { - i.Unlock() - return - } - } - } - i.lookup[key] = append(i.lookup[key], o) - i.Unlock() -} - -// Expensive, it's a map copy! -func (i *Index) AsMap() map[interface{}][]*Object { - i.RLock() - m := make(map[interface{}][]*Object) - for k, v := range i.lookup { - s := make([]*Object, len(v), len(v)) - copy(s, v) - m[k] = s - } - i.RUnlock() - return m -} - func NewObjects() *Objects { - var os Objects - os.idindex = make(map[uint32]int) - os.asmap = make(map[*Object]struct{}) - os.multiindexes = make(map[uint32]*Index) + os := Objects{ + // indexes: make(map[Attribute]*Index), + multiindexes: make(map[AttributePair]*MultiIndex), + } return &os } @@ -130,28 +83,13 @@ func (os *Objects) GetIndex(attribute Attribute) *Index { os.indexlock.Unlock() return index } - os.indexlock.RUnlock() return index - - /* - i := os.indexes[attribute] - mi := make(map[interface{}][]*Object) - for k, v := range i { - s := make([]*Object, len(v)) - i := 0 - for o, _ := range v { - s[i] = o - i++ - } - mi[k] = s - } - return mi */ } -func (os *Objects) GetMultiIndex(attribute, attribute2 Attribute) *Index { +func (os *Objects) GetMultiIndex(attribute, attribute2 Attribute) *MultiIndex { // Consistently map to the right index no matter what order they are called - if attribute < attribute2 { + if attribute > attribute2 { attribute, attribute2 = attribute2, attribute } @@ -162,7 +100,7 @@ func (os *Objects) GetMultiIndex(attribute, attribute2 Attribute) *Index { os.indexlock.RLock() // No room for index for this attribute - indexkey := uint32(attribute)<<16 | uint32(attribute2) + indexkey := AttributePair{attribute, attribute2} index, found := os.multiindexes[indexkey] if found { @@ -180,7 +118,7 @@ func (os *Objects) GetMultiIndex(attribute, attribute2 Attribute) *Index { return index } - index = &Index{} + index = &MultiIndex{} // Initialize index and add existing stuff os.refreshMultiIndex(attribute, attribute2, index) @@ -193,48 +131,41 @@ func (os *Objects) GetMultiIndex(attribute, attribute2 Attribute) *Index { } func (os *Objects) refreshIndex(attribute Attribute, index *Index) { - index.Init() + index.init() // add all existing stuff to index - for _, o := range os.asarray { + os.Iterate(func(o *Object) bool { o.Attr(attribute).Iterate(func(value AttributeValue) bool { - key := AttributeValueToIndex(value) - // Add to index - index.Add(key, o, false) - + index.Add(value, o, false) return true // continue }) - } -} - -type multiindexkey struct { - value1 interface{} - value2 interface{} + return true + }) } -func (os *Objects) refreshMultiIndex(attribute, attribute2 Attribute, index *Index) { - index.Init() +func (os *Objects) refreshMultiIndex(attribute, attribute2 Attribute, index *MultiIndex) { + index.init() // add all existing stuff to index - for _, o := range os.asarray { + os.Iterate(func(o *Object) bool { if !o.HasAttr(attribute) || !o.HasAttr(attribute2) { - continue + return true } o.Attr(attribute).Iterate(func(value AttributeValue) bool { - key := AttributeValueToIndex(value) + iv := AttributeValueToIndex(value) o.Attr(attribute2).Iterate(func(value2 AttributeValue) bool { - key2 := AttributeValueToIndex(value2) - + iv2 := AttributeValueToIndex(value2) // Add to index - index.Add(multiindexkey{key, key2}, o, false) + index.Add(iv, iv2, o, false) return true }) return true }) - } + return true + }) } func (os *Objects) SetRoot(ro *Object) { @@ -245,7 +176,7 @@ func (os *Objects) DropIndexes() { // Clear all indexes os.indexlock.Lock() os.indexes = make([]*Index, 0) - os.multiindexes = make(map[uint32]*Index) + os.multiindexes = make(map[AttributePair]*MultiIndex) os.indexlock.Unlock() } @@ -259,12 +190,11 @@ func (os *Objects) DropIndex(attribute Attribute) { } func (os *Objects) ReindexObject(o *Object, isnew bool) { - os.indexlock.RLock() - // Single attribute indexes + os.indexlock.RLock() for i, index := range os.indexes { - attribute := Attribute(i) if index != nil { + attribute := Attribute(i) o.AttrRendered(attribute).Iterate(func(value AttributeValue) bool { // If it's a string, lowercase it before adding to index, we do the same on lookups indexval := AttributeValueToIndex(value) @@ -288,9 +218,9 @@ func (os *Objects) ReindexObject(o *Object, isnew bool) { } // Multi indexes - for i, index := range os.multiindexes { - attribute := Attribute((i >> 16) & 0xffff) - attribute2 := Attribute(i & 0xffff) + for attributes, index := range os.multiindexes { + attribute := attributes.attribute1 + attribute2 := attributes.attribute2 if !o.HasAttr(attribute) || !o.HasAttr(attribute2) { continue @@ -301,7 +231,7 @@ func (os *Objects) ReindexObject(o *Object, isnew bool) { o.Attr(attribute2).Iterate(func(value2 AttributeValue) bool { key2 := AttributeValueToIndex(value2) - index.Add(multiindexkey{key, key2}, o, !isnew) + index.Add(key, key2, o, !isnew) return true }) @@ -313,30 +243,31 @@ func (os *Objects) ReindexObject(o *Object, isnew bool) { var avtiCache = cache.New(time.Second * 30) -func AttributeValueToIndex(value AttributeValue) interface{} { +func AttributeValueToIndex(value AttributeValue) AttributeValue { + if value == nil { + return nil + } if vs, ok := value.(AttributeValueString); ok { s := string(vs) if lowered, found := avtiCache.Get(s); found { - return lowered + return lowered.(AttributeValue) } - lowered := strings.ToLower(string(vs)) + lowered := AttributeValueString(strings.ToLower(string(vs))) avtiCache.Set(s, lowered, time.Second*30) return lowered } - return value.Raw() + return value } func (os *Objects) Filter(evaluate func(o *Object) bool) *Objects { result := NewObjects() - os.objectmutex.RLock() - objects := os.asarray - for _, object := range objects { + os.Iterate(func(object *Object) bool { if evaluate(object) { result.Add(object) } - } - os.objectmutex.RUnlock() + return true + }) return result } @@ -365,11 +296,9 @@ func (os *Objects) AddMerge(attrtomerge []Attribute, obs ...*Object) { // Attemps to merge the object into the objects func (os *Objects) Merge(attrtomerge []Attribute, o *Object) bool { - os.objectmutex.RLock() - if _, found := os.asmap[o]; found { + if _, found := os.idindex.Load(o.ID()); found { ui.Fatal().Msg("Object already exists in objects, so we can't merge it") } - os.objectmutex.RUnlock() // var deb int var merged bool @@ -436,7 +365,7 @@ func (os *Objects) add(o *Object) { ui.Fatal().Msg("Objects must have a unique ID") } - if _, found := os.asmap[o]; found { + if _, found := os.idindex.LoadOrStore(o.ID(), o); found { ui.Fatal().Msg("Object already exists in objects, so we can't add it") } @@ -444,26 +373,6 @@ func (os *Objects) add(o *Object) { o.setFlex(os.DefaultValues...) } - // Do chunked extensions for speed - if len(os.asarray) == cap(os.asarray) { - increase := len(os.asarray) / 8 - if increase < 1024 { - increase = 1024 - } - newarray := make([]*Object, len(os.asarray), len(os.asarray)+increase) - copy(newarray, os.asarray) - os.asarray = newarray - } - - if _, found := os.idindex[o.ID()]; found { - panic("Tried to add same object twice") - } - - // Add this to the iterator array and indexes - os.asarray = append(os.asarray, o) - os.idindex[o.ID()] = len(os.asarray) - 1 - os.asmap[o] = struct{}{} - os.ReindexObject(o, true) // Statistics @@ -481,24 +390,28 @@ func (os *Objects) Statistics() typestatistics { return os.typecount } -func (os *Objects) Slice() []*Object { - os.objectmutex.RLock() - defer os.objectmutex.RUnlock() - return os.asarray +func (os *Objects) AsSlice() []*Object { + result := make([]*Object, 0, os.Len()) + os.Iterate(func(o *Object) bool { + result = append(result, o) + return true + }) + return result } func (os *Objects) Len() int { - os.objectmutex.RLock() - defer os.objectmutex.RUnlock() - return len(os.asarray) + var count int + os.idindex.Range(func(key uint32, value *Object) bool { + count++ + return true + }) + return count } func (os *Objects) Iterate(each func(o *Object) bool) { - for _, o := range os.asarray { - if !each(o) { - break - } - } + os.idindex.Range(func(key uint32, value *Object) bool { + return each(value) + }) } func (os *Objects) IterateParallel(each func(o *Object) bool, parallelFuncs int) { @@ -522,27 +435,23 @@ func (os *Objects) IterateParallel(each func(o *Object) bool, parallelFuncs int) }() } - for i, o := range os.asarray { - if i&0x400 == 0 && stop.Load() { + var i int + os.Iterate(func(o *Object) bool { + if i&0x3ff == 0 && stop.Load() { ui.Debug().Msg("Aborting parallel iterator for Objects") - break + return false } queue <- o - } + i++ + return true + }) close(queue) wg.Wait() } func (os *Objects) FindByID(id uint32) (o *Object, found bool) { - os.objectmutex.RLock() - index, idfound := os.idindex[id] - if idfound { - o = os.asarray[index] - found = true - } - os.objectmutex.RUnlock() - return + return os.idindex.Load(id) } func (os *Objects) MergeOrAdd(attribute Attribute, value AttributeValue, flexinit ...interface{}) (*Object, bool) { @@ -612,7 +521,7 @@ func (os *Objects) FindMultiOrAdd(attribute Attribute, value AttributeValue, add } func (os *Objects) FindTwoMultiOrAdd(attribute Attribute, value AttributeValue, attribute2 Attribute, value2 AttributeValue, addifnotfound func() *Object) ([]*Object, bool) { - if attribute2 != NonExistingAttribute && attribute < attribute2 { + if attribute > attribute2 { attribute, attribute2 = attribute2, attribute value, value2 = value2, value } @@ -621,11 +530,11 @@ func (os *Objects) FindTwoMultiOrAdd(attribute Attribute, value AttributeValue, if addifnotfound == nil { if attribute2 == NonExistingAttribute { // Lookup by one attribute - matches, found := os.GetIndex(attribute).Lookup(AttributeValueToIndex(value)) + matches, found := os.GetIndex(attribute).Lookup(value) return matches, found } else { // Lookup by two attributes - matches, found := os.GetMultiIndex(attribute, attribute2).Lookup(multiindexkey{AttributeValueToIndex(value), AttributeValueToIndex(value2)}) + matches, found := os.GetMultiIndex(attribute, attribute2).Lookup(value, value2) return matches, found } } @@ -642,7 +551,7 @@ func (os *Objects) FindTwoMultiOrAdd(attribute Attribute, value AttributeValue, } } else { // Lookup by two attributes - matches, found := os.GetMultiIndex(attribute, attribute2).Lookup(multiindexkey{AttributeValueToIndex(value), AttributeValueToIndex(value2)}) + matches, found := os.GetMultiIndex(attribute, attribute2).Lookup(value, value2) if found { os.objectmutex.Unlock() return matches, found diff --git a/modules/engine/processing.go b/modules/engine/processing.go index 1692063..27bb119 100644 --- a/modules/engine/processing.go +++ b/modules/engine/processing.go @@ -2,14 +2,16 @@ package engine import ( "runtime" + "sync" + gsync "github.com/SaveTheRbtz/generic-sync-map-go" "github.com/lkarlslund/adalanche/modules/ui" ) func Merge(aos []*Objects) (*Objects, error) { var biggest, biggestcount, totalobjects int for i, caos := range aos { - loaderproduced := len(caos.Slice()) + loaderproduced := caos.Len() totalobjects += loaderproduced if loaderproduced > biggestcount { biggestcount = loaderproduced @@ -41,88 +43,120 @@ func Merge(aos []*Objects) (*Objects, error) { orphancontainer.ChildOf(globalroot) globalobjects.Add(orphancontainer) - // Iterate over all the object collections - needsmerge := make(map[*Object]struct{}) + ui.Info().Msgf("Merging %v objects into the object metaverse", totalobjects) - for _, mergeobjects := range aos { - if mergeroot := mergeobjects.Root(); mergeroot != nil { - mergeroot.ChildOf(globalroot) - } + pb := ui.ProgressBar("Merging objects from each unique source", totalobjects) - for _, o := range mergeobjects.Slice() { - needsmerge[o] = struct{}{} - } - // needsmerge = append(needsmerge, mergeobjects.Slice()...) + // To ease anti-cross-the-beams on DataSource we temporarily group each source and combine them in the end + type sourceinfo struct { + queue chan *Object + shard *Objects } - ui.Info().Msgf("Merging %v objects into the object metaverse", len(needsmerge)) + var sourcemap gsync.MapOf[string, sourceinfo] - pb := ui.ProgressBar("Merging objects from each unique source ...", len(needsmerge)) + var consumerWG, producerWG sync.WaitGroup - // To ease anti-cross-the-beams on UniqueSource we temporarily group each source and combine them in the end - sourcemap := make(map[interface{}]*Objects) - none := "" - sourcemap[none] = NewObjects() + // Iterate over all the object collections + for _, mergeobjects := range aos { + if mergeroot := mergeobjects.Root(); mergeroot != nil { + mergeroot.ChildOf(globalroot) + } - for mergeobject, _ := range needsmerge { - if mergeobject.HasAttr(DataSource) { - us := AttributeValueToIndex(mergeobject.OneAttr(DataSource)) - shard := sourcemap[us] - if shard == nil { - shard = NewObjects() - sourcemap[us] = shard + // Merge all objects into their own shard based on the DataSource attribute if any + producerWG.Add(1) + go func(os *Objects) { + nextshard := sourceinfo{ + queue: make(chan *Object, 64), + shard: NewObjects(), } - shard.AddMerge(mergeon, mergeobject) - } else { - sourcemap[none].AddMerge(mergeon, mergeobject) - } - pb.Add(1) + + os.Iterate(func(mergeobject *Object) bool { + pb.Add(1) + ds := mergeobject.OneAttr(DataSource) + if ds != nil { + ds = AttributeValueToIndex(ds) + } else { + ds = AttributeValueString("") + } + + info, loaded := sourcemap.LoadOrStore(ds.String(), nextshard) + if !loaded { + consumerWG.Add(1) + go func(shard *Objects, queue chan *Object) { + for mergeobject := range queue { + shard.AddMerge(mergeon, mergeobject) + } + consumerWG.Done() + }(info.shard, info.queue) + nextshard = sourceinfo{ + queue: make(chan *Object, 64), + shard: NewObjects(), + } + } + info.queue <- mergeobject + return true + }) + producerWG.Done() + }(mergeobjects) } + producerWG.Wait() + sourcemap.Range(func(key string, value sourceinfo) bool { + close(value.queue) + return true + }) + consumerWG.Wait() pb.Finish() var needsfinalization int - for _, sao := range sourcemap { - needsfinalization += sao.Len() - } + sourcemap.Range(func(key string, value sourceinfo) bool { + needsfinalization += value.shard.Len() + return true + }) - pb = ui.ProgressBar("Finalizing merge ...", needsfinalization) + pb = ui.ProgressBar("Finalizing merge", needsfinalization) // We're grabbing the index directly for faster processing here dnindex := globalobjects.GetIndex(DistinguishedName) - for us, usao := range sourcemap { - if us == none { - continue // not these, we'll try to merge at the very end + // Just add these. they have a DataSource so we're not merging them EXCEPT for ones with a DistinguishedName collition FML + sourcemap.Range(func(us string, usao sourceinfo) bool { + if us == "" { + return true // continue - not these, we'll try to merge at the very end } - for _, addobject := range usao.Slice() { + usao.shard.Iterate(func(addobject *Object) bool { pb.Add(1) // Here we'll deduplicate DNs, because sometimes schema and config context slips in twice if dn := addobject.OneAttr(DistinguishedName); dn != nil { if existing, exists := dnindex.Lookup(AttributeValueToIndex(dn)); exists { existing[0].AbsorbEx(addobject, true) - continue + return true } } globalobjects.Add(addobject) - } - } + return true + }) + return true + }) - for _, addobject := range sourcemap[none].Slice() { + nodatasource, _ := sourcemap.Load("") + nodatasource.shard.Iterate(func(addobject *Object) bool { pb.Add(1) // Here we'll deduplicate DNs, because sometimes schema and config context slips in twice if dn := addobject.OneAttr(DistinguishedName); dn != nil { if existing, exists := dnindex.Lookup(AttributeValueToIndex(dn)); exists { existing[0].AbsorbEx(addobject, true) - continue + return true } } globalobjects.AddMerge(mergeon, addobject) - } + return true + }) pb.Finish() - aftermergetotalobjects := len(globalobjects.Slice()) + aftermergetotalobjects := globalobjects.Len() ui.Info().Msgf("After merge we have %v objects in the metaverse (merge eliminated %v objects)", aftermergetotalobjects, totalobjects-aftermergetotalobjects) runtime.GC() @@ -144,13 +178,14 @@ func Merge(aos []*Objects) (*Objects, error) { } } } - for _, object := range globalobjects.Slice() { + globalobjects.Iterate(func(object *Object) bool { if object.Parent() == nil { object.ChildOf(orphancontainer) orphans++ } processobject(object) - } + return true + }) if orphans > 0 { ui.Warn().Msgf("Detected %v orphan objects in final results", orphans) } diff --git a/modules/integrations/activedirectory/analyze/adloader.go b/modules/integrations/activedirectory/analyze/adloader.go index 2e86bd4..d4238b1 100644 --- a/modules/integrations/activedirectory/analyze/adloader.go +++ b/modules/integrations/activedirectory/analyze/adloader.go @@ -195,9 +195,10 @@ func (ld *ADLoader) Close() ([]*engine.Objects, error) { } else { // Indicate from which domain we saw this if we have the data nb := engine.AttributeValueString(netbiosname) - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { o.SetFlex(engine.DataSource, nb) - } + return true + }) } aos = append(aos, ao) diff --git a/modules/integrations/activedirectory/analyze/analyze-ad.go b/modules/integrations/activedirectory/analyze/analyze-ad.go index c875de2..27c1761 100644 --- a/modules/integrations/activedirectory/analyze/analyze-ad.go +++ b/modules/integrations/activedirectory/analyze/analyze-ad.go @@ -106,21 +106,21 @@ func init() { return } - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { // Only for computers if o.Type() != engine.ObjectTypeComputer { - continue + return true } // ... that has LAPS installed if !o.HasAttr(activedirectory.MSmcsAdmPwdExpirationTime) { - continue + return true } // Analyze ACL sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } // Link to the machine object @@ -131,7 +131,7 @@ func init() { machine, found := ao.Find(DomainJoinedSID, engine.AttributeValueSID(machinesid)) if !found { ui.Error().Msgf("Could not locate machine for domain SID %v", machinesid) - continue + return true } for index, acl := range sd.DACL.Entries { @@ -139,57 +139,61 @@ func init() { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(machine, activedirectory.EdgeReadLAPSPassword) } } - } + return true + }) }, "Reading local admin passwords via LAPS", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { if o.Type() != engine.ObjectTypeContainer || o.OneAttrString(engine.Name) != "Machine" { - continue + return true } // Only for computers, you can't really pwn users this way p, hasparent := ao.DistinguishedParent(o) if !hasparent || p.Type() != engine.ObjectTypeGroupPolicyContainer { - continue + return true } p.EdgeTo(o, activedirectory.PartOfGPO) - } + return true + }) }, "Machine configurations that are part of a GPO", engine.BeforeMergeHigh) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { if o.Type() != engine.ObjectTypeContainer || o.OneAttrString(engine.Name) != "User" { - continue + return true } // Only for users, you can't really pwn users this way p, hasparent := ao.DistinguishedParent(o) if !hasparent || p.Type() != engine.ObjectTypeGroupPolicyContainer { - continue + return true } p.EdgeTo(o, activedirectory.PartOfGPO) - } + return true + }) }, "User configurations that are part of a GPO", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { // It's a group sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for _, acl := range sd.DACL.Entries { if acl.Type == engine.ACETYPE_ACCESS_DENIED || acl.Type == engine.ACETYPE_ACCESS_DENIED_OBJECT { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeACLContainsDeny) // Not a probability of success, this is just an indicator } } - } + return true + }) }, "Indicator for possible false positives, as the ACL contains DENY entries", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } // https://www.alsid.com/crb_article/kerberos-delegation/ // --- Citation bloc --- This is generally true, but an exception exists: positioning a Deny for the OWNER RIGHTS SID (S-1-3-4) in an object’s ACE removes the owner’s implicit control of this object’s DACL. --------------------- @@ -202,173 +206,183 @@ func init() { if !sd.Owner.IsNull() && !aclhasdeny { ao.FindOrAddAdjacentSID(sd.Owner, o).EdgeTo(o, activedirectory.EdgeOwns) } - } + return true + }) }, "Indicator that someone owns an object", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { if o.Type() == engine.ObjectTypeForeignSecurityPrincipal { - continue + return true } sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_GENERIC_ALL, engine.NullGUID, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeGenericAll) } } - } + return true + }) }, "Indicator that someone has full permissions on an object", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { if o.Type() == engine.ObjectTypeForeignSecurityPrincipal { - continue + return true } sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_GENERIC_WRITE, engine.NullGUID, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeWriteAll) } } - } + return true + }) }, "Indicator that someone can write to all attributes and do all validated writes on an object", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { if o.Type() == engine.ObjectTypeForeignSecurityPrincipal { - continue + return true } sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_DS_WRITE_PROPERTY, engine.NullGUID, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeWritePropertyAll) } } - } + return true + }) }, "Indicator that someone can write to all attributes of an object", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { if o.Type() == engine.ObjectTypeForeignSecurityPrincipal { - continue + return true } sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_DS_WRITE_PROPERTY_EXTENDED, engine.NullGUID, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeWriteExtendedAll) } } - } + return true + }) }, "Indicator that someone do all validated writes on an object", engine.BeforeMergeFinal) // https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-dtyp/c79a383c-2b3f-4655-abe7-dcbb7ce0cfbe IMPORTANT Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { if o.Type() == engine.ObjectTypeForeignSecurityPrincipal { - continue + return true } sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_WRITE_OWNER, engine.NullGUID, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeTakeOwnership) } } - } + return true + }) }, "Indicator that someone is allowed to take ownership of an object", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { if o.Type() == engine.ObjectTypeForeignSecurityPrincipal { - continue + return true } sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_WRITE_DACL, engine.NullGUID, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeWriteDACL) } } - } + return true + }) }, "Indicator that someone can change permissions on an object", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { sd, err := o.SecurityDescriptor() if o.Type() != engine.ObjectTypeAttributeSchema { - continue + return true } // FIXME - check for SYSTEM ATTRIBUTES - these can NEVER be changed if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_DS_WRITE_PROPERTY, AttributeSecurityGUIDGUID, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeWriteAttributeSecurityGUID) // Experimental, I've never run into this misconfiguration } } - } + return true + }) }, `Allows an attacker to modify the attribute security set of an attribute, promoting it to a weaker attribute set (experimental/wrong)`, engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { // Only users, computers and service accounts if o.Type() != engine.ObjectTypeUser && o.Type() != engine.ObjectTypeComputer { - continue + return true } // Check who can reset the password sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_DS_CONTROL_ACCESS, ResetPwd, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeResetPassword) } } - } + return true + }) }, "Indicator that a group or user can reset the password of an account", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { // Only managed service accounts if o.Type() != engine.ObjectTypeManagedServiceAccount && o.Type() != engine.ObjectTypeGroupManagedServiceAccount { - continue + return true } // Check who can reset the password sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_DS_READ_PROPERTY, AttributeMSDSManagedPasswordId, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeReadPasswordId) } } - } + return true + }) }, "Indicator that a group or user can read the msDS-ManagedPasswordId for use in MGSA Golden attack", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { @@ -378,17 +392,18 @@ func init() { return } - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { // Only computers and users if o.Type() != engine.ObjectTypeUser { - continue + return true } if o.Attr(activedirectory.ServicePrincipalName).Len() > 0 { o.SetValues(engine.MetaHasSPN, engine.AttributeValueInt(1)) authusers.EdgeTo(o, activedirectory.EdgeHasSPN) } - } + return true + }) }, "Indicator that a user has a ServicePrincipalName and an authenticated user can Kerberoast it", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { @@ -398,130 +413,137 @@ func init() { return } - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { // Only users if o.Type() != engine.ObjectTypeUser { - continue + return true } if uac, ok := o.AttrInt(activedirectory.UserAccountControl); ok && uac&engine.UAC_DONT_REQ_PREAUTH != 0 { everyone.EdgeTo(o, activedirectory.EdgeDontReqPreauth) } - } + return true + }) }, "Indicator that a user has \"don't require preauth\" and can be kerberoasted", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { - // Only computers and users + ao.Iterate(func(o *engine.Object) bool { + // Only users if o.Type() != engine.ObjectTypeUser { - continue + return true } sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_DS_WRITE_PROPERTY, ValidateWriteSPN, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeWriteSPN) } } - } + return true + }) }, "Indicator that a user can change the ServicePrincipalName attribute, and then Kerberoast the account", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { // Only computers and users if o.Type() != engine.ObjectTypeUser { - continue + return true } sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_DS_WRITE_PROPERTY_EXTENDED, ValidateWriteSPN, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeWriteValidatedSPN) } } - } + return true + }) }, "Indicator that a user can change the ServicePrincipalName attribute (validate write), and then Kerberoast the account", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { // Only computers if o.Type() != engine.ObjectTypeComputer { - continue + return true } sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_DS_WRITE_PROPERTY, AttributeAllowedToActOnBehalfOfOtherIdentity, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeWriteAllowedToAct) // Success rate? } } - } + return true + }) }, `Modify the msDS-AllowedToActOnBehalfOfOtherIdentity on a computer to enable any SPN enabled user to impersonate anyone else`, engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { // Only for groups if o.Type() != engine.ObjectTypeGroup { - continue + return true } // It's a group sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_DS_WRITE_PROPERTY, AttributeMember, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeAddMember) } } - } + return true + }) }, "Permission to add a member to a group", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { // Only for groups if o.Type() != engine.ObjectTypeGroup { - continue + return true } // It's a group sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_DS_WRITE_PROPERTY, AttributeSetGroupMembership, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeAddMemberGroupAttr) } } - } + return true + }) }, "Permission to add a member to a group (via attribute set)", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { // Only for groups if o.Type() != engine.ObjectTypeGroup { - continue + return true } // It's a group sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_DS_WRITE_PROPERTY_EXTENDED, ValidateWriteSelfMembership, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeAddSelfMember) } } - } + return true + }) }, "Permission to add yourself to a group", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { o.Attr(activedirectory.MSDSGroupMSAMembership).Iterate(func(msads engine.AttributeValue) bool { sd, err := engine.ParseSecurityDescriptor([]byte(msads.String())) if err == nil { @@ -533,94 +555,100 @@ func init() { } return true }) - } + return true + }) }, "Allows someone to read a password of a managed service account", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { // Only for users if o.Type() != engine.ObjectTypeUser { - continue + return true } sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_DS_WRITE_PROPERTY, AttributeAltSecurityIdentitiesGUID, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeWriteAltSecurityIdentities) } } - } + return true + }) }, "Allows an attacker to define a certificate that can be used to authenticate as the user", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { // Only for users if o.Type() != engine.ObjectTypeUser { - continue + return true } sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_DS_WRITE_PROPERTY, AttributeProfilePathGUID, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeWriteProfilePath) } } - } + return true + }) }, "Change user profile path (allows an attacker to trigger a user auth against an attacker controlled UNC path)", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { // Only for users if o.Type() != engine.ObjectTypeUser { - continue + return true } sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_DS_WRITE_PROPERTY, AttributeScriptPathGUID, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeWriteScriptPath) } } - } + return true + }) }, "Change user script path (allows an attacker to trigger a user auth against an attacker controlled UNC path)", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { o.Attr(activedirectory.MSDSHostServiceAccount).Iterate(func(dn engine.AttributeValue) bool { if targetmsa, found := ao.Find(engine.DistinguishedName, dn); found { o.EdgeTo(targetmsa, activedirectory.EdgeHasMSA) } return true }) - } + return true + }) }, "Indicates that the object has a service account in use", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { // Only for groups if o.Type() != engine.ObjectTypeUser && o.Type() != engine.ObjectTypeComputer { - continue + return true } // It's a group sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_DS_WRITE_PROPERTY, AttributeMSDSKeyCredentialLink, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeWriteKeyCredentialLink) } } - } + return true + }) }, "Allows you to write your own cert to keyCredentialLink, and then auth as that user (no password reset needed)", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { o.Attr(activedirectory.SIDHistory).Iterate(func(sidval engine.AttributeValue) bool { if sid, ok := sidval.Raw().(windowssecurity.SID); ok { target := ao.FindOrAddAdjacentSID(sid, o) @@ -628,31 +656,33 @@ func init() { } return true }) - } + return true + }) }, "Indicates that object has a SID History attribute pointing to the other object, making them the 'same' permission wise", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { if o.Type() == engine.ObjectTypeForeignSecurityPrincipal { - continue + return true } sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_DS_CONTROL_ACCESS, engine.NullGUID, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeAllExtendedRights) } } - } + return true + }) }, "Indicates that you have all extended rights", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { if o.Type() != engine.ObjectTypePKIEnrollmentService { - continue + return true } for _, cts := range o.AttrString(engine.A("certificateTemplates")) { for _, ct := range cts { @@ -664,17 +694,18 @@ func init() { } } } - } + return true + }) }, "Certificate service publishes Certificate Template", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { if o.Type() != engine.ObjectTypeCertificateTemplate { - continue + return true } sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { // https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-crtd/211ab1e3-bad6-416d-9d56-8480b42617a4 @@ -683,17 +714,18 @@ func init() { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeCertificateEnroll) } } - } + return true + }) }, "Permission to enroll into a certificate template", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { if o.Type() != engine.ObjectTypeCertificateTemplate { - continue + return true } sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { // https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-crtd/211ab1e3-bad6-416d-9d56-8480b42617a4 @@ -702,34 +734,36 @@ func init() { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeCertificateAutoEnroll) } } - } + return true + }) }, "Permission to auto-enroll into a certificate template", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_DS_VOODOO_BIT, uuid.Nil, ao) { ao.FindOrAddAdjacentSID(acl.SID, o).EdgeTo(o, activedirectory.EdgeVoodooBit) } } - } + return true + }) }, "Has the Voodoo Bit set", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { if o.Type() != engine.ObjectTypeDomainDNS { - continue + return true } if !o.HasAttr(activedirectory.SystemFlags) { - continue + return true } sd, err := o.SecurityDescriptor() if err != nil { - continue + return true } for index, acl := range sd.DACL.Entries { if sd.DACL.IsObjectClassAccessAllowed(index, o, engine.RIGHT_DS_CONTROL_ACCESS, DSReplicationSyncronize, ao) { @@ -754,20 +788,21 @@ func init() { } return true }) - } + return true + }) }, "Permissions on DomainDNS objects leading to DCsync attacks", engine.BeforeMergeFinal) Loader.AddProcessor(func(ao *engine.Objects) { // Ensure everyone has a family - for _, computeraccount := range ao.Slice() { + ao.Iterate(func(computeraccount *engine.Object) bool { if computeraccount.Type() != engine.ObjectTypeComputer { - continue + return true } sid := computeraccount.OneAttr(engine.ObjectSid) if sid == nil { ui.Error().Msgf("Computer account without SID: %v", computeraccount.DN()) - continue + return true } machine, _ := ao.FindOrAdd( DomainJoinedSID, sid, @@ -781,21 +816,23 @@ func init() { machine.EdgeTo(computeraccount, EdgeAuthenticatesAs) machine.EdgeTo(computeraccount, EdgeMachineAccount) machine.ChildOf(computeraccount) - } + + return true + }) }, "creating Machine objects (representing the machine running the OS)", engine.BeforeMerge) Loader.AddProcessor(func(ao *engine.Objects) { // Ensure everyone has a family - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { if o.Parent() != nil { - continue + return true } if o == ao.Root() { - continue + return true } if parent, found := ao.DistinguishedParent(o); found { @@ -805,11 +842,12 @@ func init() { if o.Type() == engine.ObjectTypeDomainDNS && len(dn) > 3 && strings.EqualFold("dc=", dn[:3]) { // Top of some AD we think, hook to top of browsable tree o.ChildOf(ao.Root()) - continue + return true } ui.Debug().Msgf("AD object %v (%v) has no parent :-(", o.Label(), o.DN()) } - } + return true + }) }, "applying parent/child relationships", engine.BeforeMergeHigh) @@ -856,9 +894,9 @@ func init() { }) // Apply DownLevelLogonName to relevant objects - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { if !o.HasAttr(engine.SAMAccountName) { - continue + return true } dn := o.DN() for _, domaininfo := range domains { @@ -867,7 +905,8 @@ func init() { break } } - } + return true + }) ao.DropIndex(engine.DownLevelLogonName) }, @@ -876,14 +915,14 @@ func init() { Loader.AddProcessor(func(ao *engine.Objects) { // Add domain part attribute from distinguished name to objects - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { // Only objects with a DistinguishedName if o.DN() == "" { - continue + return true } if o.HasAttr(engine.DomainContext) { - continue + return true } parts := strings.Split(o.DN(), ",") @@ -903,16 +942,17 @@ func init() { if lastpart != -1 { o.SetValues(engine.DomainContext, engine.AttributeValueString(strings.Join(parts[lastpart:], ","))) } - } + return true + }) }, "applying domain part attribute", engine.BeforeMergeLow) Loader.AddProcessor(func(ao *engine.Objects) { // Find all the AdminSDHolder containers - for _, adminsdholder := range ao.Filter(func(o *engine.Object) bool { + ao.Filter(func(o *engine.Object) bool { return strings.HasPrefix(o.OneAttrString(engine.DistinguishedName), "CN=AdminSDHolder,CN=System,") - }).Slice() { + }).Iterate(func(adminsdholder *engine.Object) bool { // We found it - so we know it can change ACLs of some objects domaincontext := adminsdholder.OneAttrString(engine.DomainContext) @@ -928,7 +968,7 @@ func init() { } } - for _, o := range ao.Filter(func(o *engine.Object) bool { + ao.Filter(func(o *engine.Object) bool { // Check if object is a group if o.Type() != engine.ObjectTypeGroup { return false @@ -939,11 +979,11 @@ func init() { return false } return true - }).Slice() { + }).Iterate(func(o *engine.Object) bool { grpsid := o.SID() if grpsid.IsNull() { - continue + return true } switch grpsid.RID() { @@ -957,30 +997,30 @@ func init() { case DOMAIN_ALIAS_RID_ADMINS: case DOMAIN_ALIAS_RID_ACCOUNT_OPS: if excluded_mask&1 != 0 { - continue + return true } case DOMAIN_ALIAS_RID_SYSTEM_OPS: if excluded_mask&2 != 0 { - continue + return true } case DOMAIN_ALIAS_RID_PRINT_OPS: if excluded_mask&4 != 0 { - continue + return true } case DOMAIN_ALIAS_RID_BACKUP_OPS: if excluded_mask&8 != 0 { - continue + return true } case DOMAIN_ALIAS_RID_REPLICATOR: default: // Not a protected group - continue + return true } // Only domain groups if grpsid.Component(2) != 21 && grpsid.Component(2) != 32 { ui.Debug().Msgf("RID match but not domain object for %v with SID %v", o.OneAttrString(engine.DistinguishedName), o.SID().String()) - continue + return true } // Apply this edge @@ -989,8 +1029,10 @@ func init() { adminsdholder.EdgeTo(target, activedirectory.EdgeOverwritesACL) return true }) - } - } + return true + }) + return true + }) }, "AdminSDHolder rights propagation indicator", engine.BeforeMerge) @@ -1046,7 +1088,7 @@ func init() { SourceSID: domainsid.String(), }, TrustInfo{}) - for _, object := range ao.Slice() { + ao.Iterate(func(object *engine.Object) bool { if rid, ok := object.AttrInt(activedirectory.PrimaryGroupID); ok { sid := object.SID() if len(sid) > 8 { @@ -1187,13 +1229,14 @@ func init() { engine.AllSchemaClasses[u] = object } }*/ - } + return true + }) }, "Active Directory objects and metadata", engine.BeforeMerge) Loader.AddProcessor(func(ao *engine.Objects) { - for _, object := range ao.Slice() { + ao.Iterate(func(object *engine.Object) bool { // We'll put the ObjectClass UUIDs in a synthetic attribute, so we can look it up later quickly (and without access to Objects) objectclasses := object.Attr(engine.ObjectClass) if objectclasses.Len() > 0 { @@ -1241,14 +1284,15 @@ func init() { engine.ObjectCategoryGUID, objectcategoryguid, engine.ObjectCategorySimple, simple, ) - } + return true + }) }, "Set ObjectCategorySimple (for Type call) to Active Directory objects", engine.BeforeMergeLow, ) Loader.AddProcessor(func(ao *engine.Objects) { - for _, object := range ao.Slice() { + ao.Iterate(func(object *engine.Object) bool { if object.HasAttrValue(engine.Name, engine.AttributeValueString("Protected Users")) && object.SID().RID() == 525 { // "Protected Users" object.EdgeIteratorRecursive(engine.In, engine.EdgeBitmap{}.Set(activedirectory.EdgeMemberOfGroup), func(source, member *engine.Object, edge engine.EdgeBitmap, depth int) bool { if member.Type() == engine.ObjectTypeComputer || member.Type() == engine.ObjectTypeUser { @@ -1257,7 +1301,8 @@ func init() { return true }) } - } + return true + }) }, "Protected users meta attribute", engine.BeforeMerge, @@ -1281,19 +1326,19 @@ func init() { } } - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { if o.HasAttr(engine.ObjectSid) && o.SID().Component(2) == 21 && !o.HasAttr(engine.DistinguishedName) && o.HasAttr(engine.DomainContext) { // An unknown SID, is it ours or from another domain? ourDomainDN := o.OneAttrString(engine.DomainContext) ourDomainSid, domainfound := domains[ourDomainDN] if !domainfound { - continue + return true } if o.SID().StripRID() == ourDomainSid { - ui.Debug().Msgf("Found a 'dangling' local SID object %v. This is either a SID from a deleted object (most likely) or hardened objects that are not readable with the account used to dump data.", o.SID()) + // ui.Debug().Msgf("Found a 'dangling' local SID object %v. This is either a SID from a deleted object (most likely) or hardened objects that are not readable with the account used to dump data.", o.SID()) } else { - ui.Debug().Msgf("Found a 'lost' foreign SID object %v, adding it as a synthetic Foreign-Security-Principal", o.SID()) + // ui.Debug().Msgf("Found a 'lost' foreign SID object %v, adding it as a synthetic Foreign-Security-Principal", o.SID()) o.SetFlex( engine.DistinguishedName, engine.AttributeValueString(o.SID().String()+",CN=ForeignSecurityPrincipals,"+ourDomainDN), engine.ObjectCategorySimple, "Foreign-Security-Principal", @@ -1301,16 +1346,17 @@ func init() { ) } } - } + return true + }) }, "Creation of synthetic Foreign-Security-Principal objects", engine.AfterMergeLow) Loader.AddProcessor(func(ao *engine.Objects) { - for _, machine := range ao.Slice() { + ao.Iterate(func(machine *engine.Object) bool { // Only for machines, you can't really pwn users this way if machine.Type() != ObjectTypeMachine { - continue + return true } // Find the computer AD object if any @@ -1324,7 +1370,7 @@ func init() { }) if computer == nil { - continue + return true } // Find all perent containers with GP links @@ -1413,20 +1459,22 @@ func init() { allowEnforcedGPOsOnly = true } } - } + return true + }) }, "Computers affected by a GPO", engine.AfterMergeLow, ) Loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { if o.HasAttr(engine.ObjectSid) && !o.HasAttr(engine.DisplayName) { if name, found := windowssecurity.KnownSIDs[o.SID().String()]; found { o.SetFlex(engine.DisplayName, name) } } - } + return true + }) }, "Adding displayName to Well-Known SID objects that are missing them", engine.AfterMergeLow) @@ -1470,7 +1518,7 @@ func init() { */ Loader.AddProcessor(func(ao *engine.Objects) { - for _, object := range ao.Slice() { + ao.Iterate(func(object *engine.Object) bool { // Object that is member of something object.Attr(activedirectory.MemberOf).Iterate(func(memberof engine.AttributeValue) bool { group, found := ao.Find(engine.DistinguishedName, memberof) @@ -1529,7 +1577,8 @@ func init() { memberobject.EdgeTo(object, activedirectory.EdgeMemberOfGroup) return true }) - } + return true + }) }, "MemberOf and Member resolution", engine.AfterMergeLow, @@ -1556,13 +1605,13 @@ func init() { ) Loader.AddProcessor(func(ao *engine.Objects) { - for _, foreign := range ao.Filter(func(o *engine.Object) bool { + ao.Filter(func(o *engine.Object) bool { return o.Type() == engine.ObjectTypeForeignSecurityPrincipal - }).Slice() { + }).Iterate(func(foreign *engine.Object) bool { sid := foreign.SID() if sid.IsNull() { ui.Error().Msgf("Found a foreign security principal with no SID %v", foreign.Label()) - continue + return true } if sid.Component(2) == 21 { if sources, found := ao.FindMulti(engine.ObjectSid, engine.AttributeValueSID(sid)); found { @@ -1575,16 +1624,17 @@ func init() { } else { ui.Warn().Msgf("Found a foreign security principal %v with an non type 21 SID %v", foreign.DN(), sid.String()) } - } + return true + }) }, "Link foreign security principals to their native objects", engine.AfterMerge, ) Loader.AddProcessor(func(ao *engine.Objects) { var warnlines int - for _, gpo := range ao.Filter(func(o *engine.Object) bool { + ao.Filter(func(o *engine.Object) bool { return o.Type() == engine.ObjectTypeGroupPolicyContainer - }).Slice() { + }).Iterate(func(gpo *engine.Object) bool { gpo.Edges(engine.In).Range(func(group *engine.Object, methods engine.EdgeBitmap) bool { groupname := group.OneAttrString(engine.SAMAccountName) if strings.Contains(groupname, "%") { @@ -1636,7 +1686,8 @@ func init() { } return true }) - } + return true + }) if warnlines > 0 { ui.Warn().Msgf("%v groups could not be resolved, this could affect analysis results", warnlines) } diff --git a/modules/integrations/localmachine/analyze/analyzers.go b/modules/integrations/localmachine/analyze/analyzers.go index be2ca1c..ab96067 100644 --- a/modules/integrations/localmachine/analyze/analyzers.go +++ b/modules/integrations/localmachine/analyze/analyzers.go @@ -6,7 +6,7 @@ import ( ) func LinkSCCM(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { if o.HasAttr(WUServer) || o.HasAttr(SCCMServer) { var hosts []string if hostname := o.OneAttrString(WUServer); hostname != "" { @@ -31,7 +31,8 @@ func LinkSCCM(ao *engine.Objects) { } } } - } + return true + }) } func init() { diff --git a/modules/integrations/localmachine/analyze/postprocessing.go b/modules/integrations/localmachine/analyze/postprocessing.go index d011a82..8beca06 100644 --- a/modules/integrations/localmachine/analyze/postprocessing.go +++ b/modules/integrations/localmachine/analyze/postprocessing.go @@ -9,7 +9,7 @@ import ( func init() { loader.AddProcessor(func(ao *engine.Objects) { - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { if o.HasAttr(activedirectory.ObjectSid) && o.HasAttr(engine.DataSource) { // We can do this with confidence as everything comes from this loader @@ -42,13 +42,14 @@ func init() { // } } } - } + return true + }) }, "Link local users and groups to machines", engine.BeforeMergeLow) loader.AddProcessor(func(ao *engine.Objects) { var warns int ln := engine.AttributeValueString(loadername) - for _, o := range ao.Slice() { + ao.Iterate(func(o *engine.Object) bool { if o.HasAttrValue(engine.DataLoader, ln) { if o.HasAttr(activedirectory.ObjectSid) { if o.Edges(engine.Out).Len()+o.Edges(engine.In).Len() == 0 { @@ -57,11 +58,12 @@ func init() { warns++ if warns > 100 { ui.Debug().Msg("Stopping warnings about graph connections, too much output") - break + return false } } } - } + return true + }) }, "Detecting broken links", engine.AfterMergeHigh, diff --git a/modules/query/execute.go b/modules/query/execute.go index 68f7cc0..e68a425 100644 --- a/modules/query/execute.go +++ b/modules/query/execute.go @@ -48,7 +48,7 @@ func Execute(q NodeFilter, ao *engine.Objects) *engine.Objects { for i, potentialIndex := range potentialindexes { index := ao.GetIndex(potentialIndex.a) - foundObjects, found := index.Lookup(engine.AttributeValueToIndex(engine.AttributeValueString(potentialIndex.match))) + foundObjects, found := index.Lookup(engine.AttributeValueString(potentialIndex.match)) if found { potentialindexes[i].results = foundObjects }