Added progressbars and engine status to UI, moved som processing task…

…s to the background

modules/analyze/cli.go

@@ -4,11 +4,8 @@ import (
 	"fmt"
 	"os/exec"
 	"runtime"
-	"runtime/debug"
-	"time"
 
 	"github.com/lkarlslund/adalanche/modules/cli"
-	"github.com/lkarlslund/adalanche/modules/dedup"
 	"github.com/lkarlslund/adalanche/modules/engine"
 	"github.com/lkarlslund/adalanche/modules/ui"
 	"github.com/spf13/cobra"
@@ -34,36 +31,15 @@ func init() {
 }
 
 func Execute(cmd *cobra.Command, args []string) error {
-	starttime := time.Now()
-
 	datapath := cmd.InheritedFlags().Lookup("datapath").Value.String()
 
+	// Process what we can in foreground, and the rest in the background
 	objs, err := engine.Run(datapath)
 	if err != nil {
 		return err
 	}
 
-	// After all this loading and merging, it's time to do release unused RAM
-	debug.FreeOSMemory()
-
-	ui.Info().Msgf("Processing done in %v", time.Since(starttime))
-
-	dedupStats := dedup.D.Statistics()
-
-	ui.Debug().Msgf("Deduplicator stats: %v items added using %v bytes in memory", dedupStats.ItemsAdded, dedupStats.BytesInMemory)
-	ui.Debug().Msgf("Deduplicator stats: %v items not allocated saving %v bytes of memory", dedupStats.ItemsSaved, dedupStats.BytesSaved)
-	ui.Debug().Msgf("Deduplicator stats: %v items removed (memory stats unavailable)", dedupStats.ItemsRemoved)
-	ui.Debug().Msgf("Deduplicator stats: %v collisions detected (first at %v objects)", dedupStats.Collisions, dedupStats.FirstCollisionDetected)
-	ui.Debug().Msgf("Deduplicator stats: %v keepalive objects added", dedupStats.KeepAliveItemsAdded)
-	ui.Debug().Msgf("Deduplicator stats: %v keepalive objects removed", dedupStats.KeepAliveItemsRemoved)
-
-	// Try to recover some memory
-	dedup.D.Flush()
-	objs.DropIndexes()
-
-	runtime.GC()
-	debug.FreeOSMemory()
-
+	// Fire up the web interface with incomplete results
 	err = WebService.Start(*bind, objs, *localhtml)
 	if err != nil {
 		return err

modules/analyze/html/adalanche.css

@@ -72,6 +72,17 @@ button, input, optgroup, select, textarea {
   display: inline-block;
 }
 
+#upperstatus {
+  position: absolute;
+  left: 50%;
+  top: 0;
+  transform: translate(-50%, 0%);
+}
+
+#progressbars {
+  width: 600px;
+}
+
 #status {
   position: absolute;
   left: 50%;
@@ -86,13 +97,6 @@ button, input, optgroup, select, textarea {
   transform: translate(-50%, 0%);
 }
 
-#xxabout {
-  position: absolute;
-  left: 50%;
-  top: 0px;
-  transform: translate(-50%, 0%);
-}
-
 #lower-left {
   position: absolute;
   left: 10px;

modules/analyze/html/custom.js

@@ -271,6 +271,58 @@ function analyze(e) {
     });
 }
 
+function refreshStatus() {
+    $.ajax({
+        type: "GET",
+        url: "/progress",
+        dataType: "json",
+        success: function (progressbars) {
+            if (progressbars.length > 0) {
+                keepProgressbars = new Set()
+                for (i in progressbars) {
+                    progressbar = progressbars[i]
+                    if (progressbar.Done) {
+                        continue
+                    }
+                    keepProgressbars.add(progressbar.ID)
+
+                    // find progressbar
+                    pb = $("#"+progressbar.ID)
+                    if (pb.length == 0 && !progressbar.Done) {
+                        $("#progressbars").append(`<div class="progress-group"><span class="progress-group-label">` + progressbar.Title + `</span><div class="progress"><div id="` + progressbar.ID + `" class="progress-bar rounded-0" role="progressbar" aria-valuemin="0" aria-valuemax="100"></div></div><span class="progress-group-label"></span></div>`)
+                        pb = $("#" + progressbar.ID)
+                    }
+
+                    // Update progressbar
+                    pb.attr("aria-valuenow", progressbar.Percent.toFixed(0))
+                    pb.css("width", progressbar.Percent.toFixed(0)+"%")
+                    pb.parent().next().html(progressbar.Percent.toFixed(2)+"%")
+                }
+                // remove old progressbars
+                $("#progressbars .progress-bar").each(function (index) {
+                    id = $(this).attr('id')
+                    if (!keepProgressbars.has(id)) {
+                        $(this).parent().parent().slideUp("slow", function () { $(this).remove(); })
+                    }
+                })
+
+                $("#progressbars").show()
+                $("#backendstatus").html("Adalanche is processing")
+            } else {
+                $("#progressbars").empty().hide()
+                $("#backendstatus").html("Adalanche backend is idle")
+            }
+        },
+        error: function (xhr, status, error) {
+            $("#backendstatus").html("Adalanche backend is offline")
+            $("#progressbars").empty().hide()
+        }
+    });
+    $()
+
+    setTimeout(refreshStatus, 1000);
+}
+
 // When we´re ready ...
 $(function () {
     // Initial GUI setup
@@ -577,5 +629,6 @@ $(function () {
         }
     });
 
+    refreshStatus();
     // End of on document loaded function
 });

modules/analyze/html/index.html

@@ -59,6 +59,11 @@
 
   <div id="cy" class="fullscreen z-10"></div>
 
+  <div id="upperstatus" class="border bg-dark p-5 shadow pointer-events-auto">
+    <div id="backendstatus" class="text-center w-full pl-20 pr-20">Loading UI ...</div>
+    <div id="progressbars"></div>
+  </div>
+
   <div id="overlay" class="fullscreen z-40 pointer-events-none">
     <div id="lower-left">
       <div id="about">
@@ -237,10 +242,10 @@
     </div>
   </div>
 
-  <!-- Absolute stuff -->
   <div id="status" class="border bg-dark p-5 shadow pointer-events-auto">
   </div>
 
+
   <div id="outerquery" class="card border mr-0 mb-0 mt-0 p-0 pointer-events-auto">
     <div id="querypop" class="text-center pl-20 pr-20">LDAP Query</div>
     <div id="querydiv" class="p-10">

modules/analyze/webservicefuncs.go

@@ -867,6 +867,42 @@ func analysisfuncs(ws *webservice) {
 		w.Write(data)
 	})
 
+	type ProgressReport struct {
+		ID             uuid.UUID
+		Title          string
+		Current, Total int64
+		Percent        float32
+		Done           bool
+		StartTime      time.Time
+	}
+
+	ws.Router.HandleFunc("/progress", func(w http.ResponseWriter, r *http.Request) {
+		pbs := ui.GetProgressBars()
+		pbr := make([]ProgressReport, len(pbs))
+		for i, pb := range pbs {
+			pbr[i] = ProgressReport{
+				ID:        pb.ID,
+				Title:     pb.Title,
+				Current:   pb.Current,
+				Total:     pb.Total,
+				Percent:   pb.Percent,
+				Done:      pb.Done,
+				StartTime: pb.Started,
+			}
+		}
+
+		sort.Slice(pbr, func(i, j int) bool {
+			return pbr[i].StartTime.Before(pbr[j].StartTime)
+		})
+
+		data, err := json.MarshalIndent(pbr, "", "  ")
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+			return
+		}
+		w.Write(data)
+	})
+
 	// Saved preferences
 	var prefs Prefs
 	err := prefs.Load()

modules/engine/analyzeobjects.go

@@ -92,7 +92,10 @@ func AnalyzeObjects(opts AnalyzeObjectsOptions) (pg Graph) {
 		}
 	}
 
+	pb := ui.ProgressBar("Analyzing graph", opts.MaxDepth)
+	pb.Add(-1)
 	for opts.MaxDepth >= processinground {
+		pb.Add(1)
 		if processinground == 2 {
 			detectedges = opts.MethodsM
 			detectobjecttypes = nil
@@ -275,6 +278,9 @@ func AnalyzeObjects(opts AnalyzeObjectsOptions) (pg Graph) {
 			}
 		}
 	}
+	pb.Finish()
+
+	pb = ui.ProgressBar("Removing filtered nodes", len(connectionsmap))
 
 	// Remove outer end nodes that are invalid
 	detectobjecttypes = nil
@@ -301,11 +307,13 @@ func AnalyzeObjects(opts AnalyzeObjectsOptions) (pg Graph) {
 				if opts.MethodsL.Intersect(detectedmethods).Count() == 0 {
 					// No matches on LastMethods
 					delete(connectionsmap, pair)
+					pb.Add(1)
 					removed++
 				} else if detectobjecttypes != nil {
 					if _, found := detectobjecttypes[pair.Target.Type()]; !found {
 						// No matches on LastMethods
 						delete(connectionsmap, pair)
+						pb.Add(1)
 						removed++
 					}
 				}
@@ -320,6 +328,7 @@ func AnalyzeObjects(opts AnalyzeObjectsOptions) (pg Graph) {
 
 		weremovedsomething = true
 	}
+	pb.Finish()
 
 	// PruneIslands
 	var prunedislands int

modules/engine/edgeconnections.go

@@ -18,7 +18,7 @@ var globalEdgeConnectionsLock sync.Mutex // Ugly but it will do
 func (ec *EdgeConnections) StringMap() map[string]string {
 	result := make(map[string]string)
 	ec.RangeID(func(id uint32, eb EdgeBitmap) bool {
-		result[IDtoOBject(id).Label()] = eb.JoinedString()
+		result[IDtoObject(id).Label()] = eb.JoinedString()
 		return true
 	})
 	return result
@@ -27,7 +27,7 @@ func (ec *EdgeConnections) StringMap() map[string]string {
 // Thread safe range
 func (ec *EdgeConnections) Range(rf func(*Object, EdgeBitmap) bool) {
 	ec.ecm.Range(func(id uint32, eb EdgeBitmap) bool {
-		return rf(IDtoOBject(id), eb)
+		return rf(IDtoObject(id), eb)
 	})
 }
 

modules/engine/object.go

@@ -974,7 +974,7 @@ func (o *Object) edgeIteratorRecursiveID(direction EdgeDirection, edgeMatch Edge
 			if !edgeMatches.IsBlank() {
 				appliedTo[targetid] = struct{}{}
 				if af(o.ID(), targetid, edgeMatches, depth) {
-					IDtoOBject(targetid).edgeIteratorRecursiveID(direction, edgeMatch, af, appliedTo, depth+1)
+					IDtoObject(targetid).edgeIteratorRecursiveID(direction, edgeMatch, af, appliedTo, depth+1)
 				}
 			}
 		}

modules/engine/objectindex.go

@@ -39,7 +39,7 @@ func onAddObject(newObject *Object) {
 	idToObjectLock.Unlock()
 }
 
-func IDtoOBject(id uint32) *Object {
+func IDtoObject(id uint32) *Object {
 	idToObjectLock.RLock()
 	objectPtr := atomic.LoadUintptr(&idToObject[id])
 	if objectPtr == 0 {

modules/engine/pwnanalyzers.go

@@ -2,6 +2,8 @@ package engine
 
 import (
 	"sync"
+
+	"github.com/lkarlslund/adalanche/modules/ui"
 )
 
 //go:generate go run github.com/dmarkham/enumer -type=ProcessPriority -output enums.go
@@ -42,7 +44,7 @@ func (l LoaderID) AddProcessor(pf ProcessorFunc, description string, priority Pr
 }
 
 // LoaderID = wildcard
-func Process(ao *Objects, cb ProgressCallbackFunc, l LoaderID, priority ProcessPriority) error {
+func Process(ao *Objects, statustext string, l LoaderID, priority ProcessPriority) error {
 	var priorityProcessors []ppfInfo
 	for _, potentialProcessor := range registeredProcessors {
 		if potentialProcessor.loader == l || l == -1 {
@@ -60,19 +62,18 @@ func Process(ao *Objects, cb ProgressCallbackFunc, l LoaderID, priority ProcessP
 	}
 
 	// We need to process this many objects
-	cb(0, total)
-
+	pb := ui.ProgressBar(statustext, total)
 	var wg sync.WaitGroup
-
 	for _, processor := range priorityProcessors {
 		wg.Add(1)
 		go func(ppf ppfInfo) {
 			ppf.pf(ao)
-			cb(-aoLen, 0)
+			pb.Add(aoLen)
 			wg.Done()
 		}(processor)
 	}
 	wg.Wait()
+	pb.Finish()
 
 	return nil
 }