chore: sort targets by the remaining slots

cmd/Balancer.go

@@ -34,21 +34,15 @@ func (self *Balancer) Action(cliContext *cli.Context) error {
 		},
 	})
 
-	// loadBalancer.RegisterTarget(&loadbalancer.LlamaCppTarget{
-	// 	LlamaCppClient: &llamacpp.LlamaCppClient{
-	// 		HttpClient: http.DefaultClient,
-	// 		LlamaCppConfiguration: &llamacpp.LlamaCppConfiguration{
-	// 			HttpAddress: &netcfg.HttpAddressConfiguration{
-	// 				Host:   "127.0.0.1",
-	// 				Port:   8089,
-	// 				Scheme: "http",
-	// 			},
-	// 		},
-	// 	},
-	// 	LlamaCppHealthStatus: &llamacpp.LlamaCppHealthStatus{
-	// 		SlotsIdle: 10,
-	// 	},
-	// })
+	loadBalancer.RegisterTarget(&loadbalancer.LlamaCppTargetConfiguration{
+		LlamaCppConfiguration: &llamacpp.LlamaCppConfiguration{
+			HttpAddress: &netcfg.HttpAddressConfiguration{
+				Host:   "127.0.0.1",
+				Port:   8089,
+				Scheme: "http",
+			},
+		},
+	})
 
 	managementServer := &management.Server{
 		ManagementServerConfiguration: self.ManagementServerConfiguration,

go.mod

@@ -3,7 +3,6 @@ module github.com/distantmagic/paddler
 go 1.21.8
 
 require (
-	github.com/emirpasic/gods/v2 v2.0.0-alpha
 	github.com/hashicorp/go-hclog v1.6.2
 	github.com/stretchr/testify v1.9.0
 	github.com/urfave/cli/v2 v2.27.2

go.sum

@@ -3,8 +3,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/emirpasic/gods/v2 v2.0.0-alpha h1:dwFlh8pBg1VMOXWGipNMRt8v96dKAIvBehtCt6OtunU=
-github.com/emirpasic/gods/v2 v2.0.0-alpha/go.mod h1:W0y4M2dtBB9U5z3YlghmpuUhiaZT2h6yoeE+C1sCp6A=
 github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
 github.com/hashicorp/go-hclog v1.6.2 h1:NOtoftovWkDheyUM/8JW3QMiXyxJK3uHRK7wV04nD2I=

llamacpp/LlamaCppHealthStatus.go

@@ -6,7 +6,3 @@ type LlamaCppHealthStatus struct {
 	SlotsProcessing uint                     `json:"slots_processing"`
 	Error           error                    `json:"-"`
 }
-
-func (self *LlamaCppHealthStatus) Less(other *LlamaCppHealthStatus) bool {
-	return self.SlotsIdle < other.SlotsIdle
-}

loadbalancer/LlamaCppTarget.go

@@ -9,5 +9,5 @@ type LlamaCppTarget struct {
 }
 
 func (self *LlamaCppTarget) Less(other *LlamaCppTarget) bool {
-	return self.LlamaCppHealthStatus.Less(other.LlamaCppHealthStatus)
+	return self.LlamaCppHealthStatus.SlotsIdle < other.LlamaCppHealthStatus.SlotsIdle
 }

loadbalancer/LlamaCppTargetHeap.go

@@ -0,0 +1,30 @@
+package loadbalancer
+
+type LlamaCppTargetHeap []*LlamaCppTarget
+
+func (self LlamaCppTargetHeap) Len() int {
+	return len(self)
+}
+
+func (self LlamaCppTargetHeap) Less(i, j int) bool {
+	// inverse comparison, because we want the target with the most idle slots
+	// to be at the top
+	return !self[i].Less(self[j])
+}
+
+func (self LlamaCppTargetHeap) Swap(i, j int) {
+	self[i], self[j] = self[j], self[i]
+}
+
+func (self *LlamaCppTargetHeap) Push(x any) {
+	*self = append(*self, x.(*LlamaCppTarget))
+}
+
+func (self *LlamaCppTargetHeap) Pop() any {
+	old := *self
+	n := len(old)
+	x := old[n-1]
+	*self = old[0 : n-1]
+
+	return x
+}

loadbalancer/LoadBalancer.go

@@ -1,25 +1,29 @@
 package loadbalancer
 
 import (
+	"container/heap"
 	"errors"
 	"net/http"
 	"net/url"
 
 	"github.com/distantmagic/paddler/llamacpp"
-	"github.com/emirpasic/gods/v2/trees/binaryheap"
 	"github.com/hashicorp/go-hclog"
 )
 
+var (
+	ErrorNoTargetsAvailable = errors.New("no targets available")
+)
+
 type LoadBalancer struct {
 	Logger  hclog.Logger
-	targets *binaryheap.Heap[*LlamaCppTarget]
+	targets *LlamaCppTargetHeap
 }
 
 func (self *LoadBalancer) Balance(request *http.Request) (*url.URL, error) {
-	headTarget, ok := self.targets.Peek()
+	headTarget := (*self.targets)[0]
 
-	if !ok || headTarget == nil {
-		return nil, errors.New("no targets available")
+	if headTarget == nil {
+		return nil, ErrorNoTargetsAvailable
 	}
 
 	targetUrl := headTarget.
@@ -29,14 +33,19 @@ func (self *LoadBalancer) Balance(request *http.Request) (*url.URL, error) {
 		BuildUrlWithPath("")
 
 	headTarget.LlamaCppHealthStatus.SlotsIdle -= 1
+	heap.Fix(self.targets, 0)
 
-	self.Logger.Debug("balancing", "target", targetUrl)
+	self.Logger.Debug(
+		"balancing",
+		"target", targetUrl,
+		"slots", headTarget.LlamaCppHealthStatus.SlotsIdle,
+	)
 
 	return targetUrl, nil
 }
 
 func (self *LoadBalancer) RegisterTarget(targetConfiguration *LlamaCppTargetConfiguration) {
-	self.targets.Push(&LlamaCppTarget{
+	heap.Push(self.targets, &LlamaCppTarget{
 		LlamaCppClient: &llamacpp.LlamaCppClient{
 			HttpClient:            http.DefaultClient,
 			LlamaCppConfiguration: targetConfiguration.LlamaCppConfiguration,

loadbalancer/NewLoadBalancer.go

@@ -1,15 +1,20 @@
 package loadbalancer
 
 import (
-	"github.com/emirpasic/gods/v2/trees/binaryheap"
+	"container/heap"
+
 	"github.com/hashicorp/go-hclog"
 )
 
 func NewLoadBalancer(
 	logger hclog.Logger,
 ) *LoadBalancer {
+	targetHeap := &LlamaCppTargetHeap{}
+
+	heap.Init(targetHeap)
+
 	return &LoadBalancer{
 		Logger:  logger,
-		targets: binaryheap.NewWith[*LlamaCppTarget](LlamaCppTargetComparator),
+		targets: targetHeap,
 	}
 }

main.go

@@ -52,7 +52,7 @@ func main() {
 					},
 					&cli.UintFlag{
 						Name:        "management-port",
-						Value:       8088,
+						Value:       8085,
 						Destination: &balancer.ManagementServerConfiguration.HttpAddress.Port,
 					},
 					&cli.StringFlag{
@@ -67,7 +67,7 @@ func main() {
 					},
 					&cli.UintFlag{
 						Name:        "reverseproxy-port",
-						Value:       8087,
+						Value:       8086,
 						Destination: &balancer.ReverseProxyConfiguration.HttpAddress.Port,
 					},
 					&cli.StringFlag{