dashboard: optimize mongodb queries

Fixes #6

dashboard.go

@@ -4,14 +4,17 @@ import (
 	"context"
 	"fmt"
 	"html/template"
+	"log"
 	"net/http"
-	"sort"
+	"runtime"
+	"sync"
 	"time"
 
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
+	"golang.org/x/sync/errgroup"
 )
 
 // dashboard returns a simple dashboard view to view all existing statistics.
@@ -23,8 +26,9 @@ func dashboard(w http.ResponseWriter, r *http.Request) {
 		}
 		http.Error(w, fmt.Sprintf("bad request: %v", err), http.StatusBadRequest)
 	}()
+	wait := 60 * time.Second
 
-	ctx, cancel := context.WithTimeout(r.Context(), time.Second*10)
+	ctx, cancel := context.WithTimeout(r.Context(), wait)
 	defer cancel()
 
 	cols, err := db.Database(dbname).ListCollectionNames(ctx, bson.D{})
@@ -42,67 +46,84 @@ func dashboard(w http.ResponseWriter, r *http.Request) {
 		Records []record
 	}
 
-	var all []records
+	all := make([]records, 0, len(cols))
+	mu := sync.Mutex{}
+
+	g, ctx := errgroup.WithContext(ctx)
+	g.SetLimit(runtime.NumCPU())
 	for _, hostname := range cols {
-		col := db.Database(dbname).Collection(hostname)
-		// mongodb query:
-		//
-		// db.getCollection('blog.changkun.de').aggregate([
-		// {"$group": {
-		//     _id: {path: "$path", ip:"$ip"},
-		//     count: {"$sum": 1}}
-		// },
-		// {"$group": {
-		//     _id: "$_id.path",
-		//     uv: {$sum: 1},
-		//     pv: {$sum: "$count"}}
-		// }])
-		p := mongo.Pipeline{
-			bson.D{
-				primitive.E{
-					Key: "$group", Value: bson.M{
-						"_id":   bson.M{"path": "$path", "ip": "$ip"},
-						"count": bson.M{"$sum": 1},
+		hostname := hostname
+		g.Go(func() error {
+			start := time.Now()
+			defer func() {
+				log.Printf("running for host %v took %v", hostname, time.Since(start))
+			}()
+
+			col := db.Database(dbname).Collection(hostname)
+			// mongodb query:
+			//
+			// db.getCollection('golang.design').aggregate([
+			// {"$group": {
+			//     _id: {path: "$path", ip:"$ip"},
+			//     count: {"$sum": 1}}
+			// },
+			// {"$group": {
+			//     _id: "$_id.path",
+			//     uv: {$sum: 1},
+			//     pv: {$sum: "$count"}}
+			// },
+			// {"$sort": {'pv': -1, 'uv': -1}}], { allowDiskUse: true })
+			//
+			// TODO: currently golang.design is the slowest query and should
+			// be further optimized. Maybe batched queries?
+			p := mongo.Pipeline{
+				bson.D{
+					primitive.E{
+						Key: "$group", Value: bson.M{
+							"_id":   bson.M{"path": "$path", "ip": "$ip"},
+							"count": bson.M{"$sum": 1},
+						},
 					},
 				},
-			},
-			bson.D{
-				primitive.E{
-					Key: "$group", Value: bson.M{
-						"_id": "$_id.path",
-						"uv":  bson.M{"$sum": 1},
-						"pv":  bson.M{"$sum": "$count"},
+				bson.D{
+					primitive.E{
+						Key: "$group", Value: bson.M{
+							"_id": "$_id.path",
+							"uv":  bson.M{"$sum": 1},
+							"pv":  bson.M{"$sum": "$count"},
+						},
 					},
 				},
-			},
-		}
-		opts := options.Aggregate().SetMaxTime(10 * time.Second)
-		var cur *mongo.Cursor
-		cur, err = col.Aggregate(ctx, p, opts)
-		if err != nil {
-			err = fmt.Errorf("failed to count visit: %w", err)
-			return
-		}
-		var results []record
-		err = cur.All(ctx, &results)
-		if err != nil {
-			err = fmt.Errorf("failed to count visit: %w", err)
-			return
-		}
-		all = append(all, records{
-			Host:    hostname,
-			Records: results,
-		})
-	}
-
-	for idx := range all {
-		sort.Slice(all[idx].Records, func(i, j int) bool {
-			if all[idx].Records[i].PV > all[idx].Records[j].PV {
-				return true
+				bson.D{
+					primitive.E{Key: "$sort", Value: bson.M{"pv": -1, "uv": -1}},
+				},
+			}
+			opts := options.Aggregate().SetMaxTime(wait).SetAllowDiskUse(true)
+			var cur *mongo.Cursor
+			cur, err = col.Aggregate(ctx, p, opts)
+			if err != nil {
+				err = fmt.Errorf("failed to count visit: %w", err)
+				return err
 			}
-			return all[idx].Records[i].UV > all[idx].Records[j].UV
+			var results []record
+			err = cur.All(ctx, &results)
+			if err != nil {
+				err = fmt.Errorf("failed to count visit: %w", err)
+				return err
+			}
+
+			mu.Lock()
+			all = append(all, records{
+				Host:    hostname,
+				Records: results,
+			})
+			mu.Unlock()
+			return nil
 		})
 	}
+	if err = g.Wait(); err != nil {
+		return
+	}
 
 	t, err := template.ParseFS(publicFS, "dashboard.html")
 	if err != nil {

public/dashboard.html

@@ -33,11 +33,17 @@
 </head>
 <body>
 <div id="app">
-<h1>changkun.de</h1>
-<h5><a href="https://changkun.de/s/urlstat">URLstat dashboard</a></h5>
+<h1><a href="https://changkun.de/s/urlstat">URLstat dashboard</a></h1>
+<h2>List of Hosts</h2>
+<ul>
+  {{range .All}}
+  <li><a href="#{{.Host}}">{{.Host}}</a></li>
+  {{end}}
+</ul>
+
 
 {{range .All}}
-<p><strong>{{.Host}}</strong></p>
+<h2 id="{{.Host}}"><strong>{{.Host}}</strong></h2>
 <table class="table">
 <tr><th>PV/UV</th><th>PATH</th></tr>
 {{range .Records}}