Chore: Refactor tool indexing

Signed-off-by: Daishan Peng <daishan@acorn.io>

apiclient/types/toolreference.go

@@ -15,19 +15,22 @@ const (
 type ToolReferenceManifest struct {
 	Name      string            `json:"name"`
 	ToolType  ToolReferenceType `json:"toolType"`
+	Commit    string            `json:"commit,omitempty"`
 	Reference string            `json:"reference,omitempty"`
 	Active    bool              `json:"active,omitempty"`
 }
 
 type ToolReference struct {
 	Metadata
 	ToolReferenceManifest
-	Resolved    bool              `json:"resolved,omitempty"`
-	Error       string            `json:"error,omitempty"`
-	Builtin     bool              `json:"builtin,omitempty"`
-	Description string            `json:"description,omitempty"`
-	Credentials []string          `json:"credentials,omitempty"`
-	Params      map[string]string `json:"params,omitempty"`
+	Resolved       bool              `json:"resolved,omitempty"`
+	Error          string            `json:"error,omitempty"`
+	Builtin        bool              `json:"builtin,omitempty"`
+	Description    string            `json:"description,omitempty"`
+	Credentials    []string          `json:"credentials,omitempty"`
+	Params         map[string]string `json:"params,omitempty"`
+	Bundle         bool              `json:"bundle,omitempty"`
+	BundleToolName string            `json:"bundleToolName,omitempty"`
 }
 
 type ToolReferenceList List[ToolReference]

pkg/api/handlers/toolreferences.go

@@ -12,6 +12,7 @@ import (
 	"github.com/obot-platform/obot/apiclient/types"
 	"github.com/obot-platform/obot/pkg/api"
 	v1 "github.com/obot-platform/obot/pkg/storage/apis/obot.obot.ai/v1"
+	"github.com/obot-platform/obot/pkg/tool"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -33,10 +34,13 @@ func convertToolReference(toolRef v1.ToolReference) types.ToolReference {
 			Name:      toolRef.Name,
 			ToolType:  toolRef.Spec.Type,
 			Reference: toolRef.Spec.Reference,
+			Commit:    toolRef.Status.Commit,
 		},
-		Builtin:  toolRef.Spec.Builtin,
-		Error:    toolRef.Status.Error,
-		Resolved: toolRef.Generation == toolRef.Status.ObservedGeneration,
+		Builtin:        toolRef.Spec.Builtin,
+		Bundle:         toolRef.Spec.Bundle,
+		BundleToolName: toolRef.Spec.BundleToolName,
+		Error:          toolRef.Status.Error,
+		Resolved:       toolRef.Generation == toolRef.Status.ObservedGeneration,
 	}
 	if toolRef.Spec.Active == nil {
 		tf.Active = true
@@ -123,22 +127,22 @@ func (a *ToolReferenceHandler) Create(req api.Context) (err error) {
 		return apierrors.NewBadRequest(fmt.Sprintf("invalid tool type %s", newToolReference.ToolType))
 	}
 
-	toolRef := &v1.ToolReference{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      newToolReference.Name,
-			Namespace: req.Namespace(),
-		},
-		Spec: v1.ToolReferenceSpec{
-			Type:      newToolReference.ToolType,
-			Reference: newToolReference.Reference,
-		},
+	toolRefs, err := tool.ResolveToolReferences(req.Context(), a.gptscript, newToolReference.Name, newToolReference.Reference, false, newToolReference.ToolType)
+	if err != nil {
+		return apierrors.NewBadRequest(fmt.Sprintf("failed to resolve tool references for %s: %v", newToolReference.Reference, err))
 	}
 
-	if err = req.Create(toolRef); err != nil {
-		return err
+	if len(toolRefs) == 0 {
+		return apierrors.NewBadRequest(fmt.Sprintf("no tool references found for %s", newToolReference.Reference))
+	}
+
+	for _, toolRef := range toolRefs {
+		if err := req.Create(toolRef); err != nil && !apierrors.IsAlreadyExists(err) {
+			return apierrors.NewInternalError(fmt.Errorf("failed to create tool reference %s: %w", toolRef.Name, err))
+		}
 	}
 
-	return req.Write(convertToolReference(*toolRef))
+	return req.Write(convertToolReference(*toolRefs[0]))
 }
 
 func (a *ToolReferenceHandler) Delete(req api.Context) error {

pkg/controller/data/agent.yaml

@@ -23,11 +23,11 @@ spec:
       modify the files in the workspace. Obot collaborates with the user on the files in the workspace.
     alias: obot
     tools:
-    - workspace-files
-    - time
-    - knowledge
-    - database
-    - tasks
+      - workspace-files
+      - time-bundle
+      - knowledge
+      - database
+      - tasks
     defaultThreadTools:
-    - images-bundle
-    - google-search-bundle
+      - images-bundle
+      - google-search-bundle

pkg/controller/handlers/toolreference/toolreference.go

@@ -23,6 +23,7 @@ import (
 	"github.com/obot-platform/obot/pkg/gateway/server/dispatcher"
 	v1 "github.com/obot-platform/obot/pkg/storage/apis/obot.obot.ai/v1"
 	"github.com/obot-platform/obot/pkg/system"
+	"github.com/obot-platform/obot/pkg/tool"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -83,78 +84,19 @@ func (h *Handler) toolsToToolReferences(ctx context.Context, toolType types.Tool
 		if ref, ok := strings.CutPrefix(entry.Reference, "./"); ok {
 			entry.Reference = registryURL + "/" + ref
 		}
+		if !h.supportDocker && name == system.ShellTool {
+			continue
+		}
 
-		if entry.All {
-			prg, err := h.gptClient.LoadFile(ctx, "* from "+entry.Reference)
-			if err != nil {
-				log.Errorf("Failed to load tool %s: %v", entry.Reference, err)
-				continue
-			}
+		toolRefs, err := tool.ResolveToolReferences(ctx, h.gptClient, name, entry.Reference, true, toolType)
+		if err != nil {
+			log.Errorf("Failed to resolve tool references for %s: %v", entry.Reference, err)
+			continue
+		}
 
-			tool := prg.ToolSet[prg.EntryToolID]
-			if isValidTool(tool) {
-				toolName := tool.Name
-				if tool.MetaData["bundle"] == "true" {
-					toolName = "bundle"
-				}
-				result = append(result, &v1.ToolReference{
-					ObjectMeta: metav1.ObjectMeta{
-						Name:        normalize(name, toolName),
-						Namespace:   system.DefaultNamespace,
-						Finalizers:  []string{v1.ToolReferenceFinalizer},
-						Annotations: annotations,
-					},
-					Spec: v1.ToolReferenceSpec{
-						Type:      toolType,
-						Reference: entry.Reference,
-						Builtin:   true,
-					},
-				})
-			}
-			for _, peerToolID := range tool.LocalTools {
-				// If this is the entry tool, then we already added it or skipped it above.
-				if peerToolID == prg.EntryToolID {
-					continue
-				}
-
-				peerTool := prg.ToolSet[peerToolID]
-				if isValidTool(peerTool) {
-					toolName := peerTool.Name
-					if peerTool.MetaData["bundle"] == "true" {
-						toolName += "-bundle"
-					}
-					result = append(result, &v1.ToolReference{
-						ObjectMeta: metav1.ObjectMeta{
-							Name:        normalize(name, toolName),
-							Namespace:   system.DefaultNamespace,
-							Finalizers:  []string{v1.ToolReferenceFinalizer},
-							Annotations: annotations,
-						},
-						Spec: v1.ToolReferenceSpec{
-							Type:      toolType,
-							Reference: fmt.Sprintf("%s from %s", peerTool.Name, entry.Reference),
-							Builtin:   true,
-						},
-					})
-				}
-			}
-		} else {
-			if !h.supportDocker && name == system.ShellTool {
-				continue
-			}
-			result = append(result, &v1.ToolReference{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:        name,
-					Namespace:   system.DefaultNamespace,
-					Finalizers:  []string{v1.ToolReferenceFinalizer},
-					Annotations: annotations,
-				},
-				Spec: v1.ToolReferenceSpec{
-					Type:      toolType,
-					Reference: entry.Reference,
-					Builtin:   true,
-				},
-			})
+		for _, toolRef := range toolRefs {
+			toolRef.Annotations = annotations
+			result = append(result, toolRef)
 		}
 	}
 
@@ -251,6 +193,7 @@ func (h *Handler) Populate(req router.Request, resp router.Response) error {
 	toolRef.Status.LastReferenceCheck = metav1.Now()
 	toolRef.Status.ObservedGeneration = toolRef.Generation
 	toolRef.Status.Reference = toolRef.Spec.Reference
+	toolRef.Status.Commit = ""
 	toolRef.Status.Tool = nil
 	toolRef.Status.Error = ""
 
@@ -269,6 +212,9 @@ func (h *Handler) Populate(req router.Request, resp router.Response) error {
 		Metadata:    tool.MetaData,
 		Params:      map[string]string{},
 	}
+	if tool.Source.Repo != nil {
+		toolRef.Status.Commit = tool.Source.Repo.Revision
+	}
 	if tool.Arguments != nil {
 		for name, param := range tool.Arguments.Properties {
 			if param.Value != nil {
@@ -502,3 +448,17 @@ func (h *Handler) CleanupModelProvider(req router.Request, _ router.Response) er
 func modelName(modelProviderName, modelName string) string {
 	return name.SafeConcatName(system.ModelPrefix, modelProviderName, fmt.Sprintf("%x", sha256.Sum256([]byte(modelName))))
 }
+
+func resolveToolReferenceName(toolType types.ToolReferenceType, isBundle bool, isCapability bool, toolName string, subToolName string) string {
+	if toolType == types.ToolReferenceTypeTool {
+		if isBundle {
+			if isCapability {
+				return toolName
+			}
+			return toolName + "-bundle"
+		}
+		return normalize(toolName, subToolName)
+	}
+
+	return toolName
+}

pkg/storage/apis/obot.obot.ai/v1/toolreference.go

@@ -49,11 +49,13 @@ func (in *ToolReference) GetColumns() [][]string {
 }
 
 type ToolReferenceSpec struct {
-	Type         types.ToolReferenceType `json:"type,omitempty"`
-	Builtin      bool                    `json:"builtin,omitempty"`
-	Reference    string                  `json:"reference,omitempty"`
-	Active       *bool                   `json:"active,omitempty"`
-	ForceRefresh metav1.Time             `json:"forceRefresh,omitempty"`
+	Type           types.ToolReferenceType `json:"type,omitempty"`
+	Builtin        bool                    `json:"builtin,omitempty"`
+	Reference      string                  `json:"reference,omitempty"`
+	Active         *bool                   `json:"active,omitempty"`
+	Bundle         bool                    `json:"bundle,omitempty"`
+	BundleToolName string                  `json:"bundleToolName,omitempty"`
+	ForceRefresh   metav1.Time             `json:"forceRefresh,omitempty"`
 }
 
 type ToolShortDescription struct {
@@ -70,6 +72,7 @@ type ToolShortDescription struct {
 
 type ToolReferenceStatus struct {
 	Reference          string                `json:"reference,omitempty"`
+	Commit             string                `json:"commit,omitempty"`
 	ObservedGeneration int64                 `json:"observedGeneration,omitempty"`
 	LastReferenceCheck metav1.Time           `json:"lastReferenceCheck,omitempty"`
 	Tool               *ToolShortDescription `json:"tool,omitempty"`

tools/devmode-kubeconfig

@@ -1,18 +1,34 @@
 apiVersion: v1
 clusters:
 - cluster:
-    server: https://localhost:8443
     insecure-skip-tls-verify: true
+    server: https://localhost:8443
   name: local-cluster
+- cluster:
+    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVMRENDQXBTZ0F3SUJBZ0lRTmxGZkZseVJxU1NsWUlZbG5oQ2lJekFOQmdrcWhraUc5dzBCQVFzRkFEQXYKTVMwd0t3WURWUVFERXlSaVl6QmtOak0zWmkxaE0yVTNMVFF4WXpNdFlqQTJZUzAxTkdVMVpUSmtabVUxTW1ZdwpJQmNOTWpVd01UQTRNak13T0RNMFdoZ1BNakExTlRBeE1ESXdNREE0TXpSYU1DOHhMVEFyQmdOVkJBTVRKR0pqCk1HUTJNemRtTFdFelpUY3ROREZqTXkxaU1EWmhMVFUwWlRWbE1tUm1aVFV5WmpDQ0FhSXdEUVlKS29aSWh2Y04KQVFFQkJRQURnZ0dQQURDQ0FZb0NnZ0dCQUsrYlVTRDgvQW1nclJldkNYc1E4TVZqVi85VmxRdDd4UmFvOEpHVQpBc1VVQ1JsZVZCUjgxdWZwWEtSWUlvZmcrZXN4cFhMZzNWR0lmM2ovK2ZPSVFFSG5LVkZwOFBES293TG1QU2FJCnVPdnN0NnViVFJ1MUI5Z0orNk5GZXFVQTZ0VEh4emR4b2gyTzFKY2ZxRWhlT0xLazhhTUpEUEtEbjBWeHJFQUoKK3oyMWRWQURIV3hEMWg5MWRkaEtiSlBmSkpRQWpnN05TbmtHamI3Nm5qcDhYY2J5UnZkU3VxV3NrWTVySTRsNgpqcFVjeVNjcEZhOHdtOGYvYkViaitTVUxxOGY2MDJHVGxja3dBMGtvTkw0alI3QnpIN2l4VE5KUXJrZkc0dkQ4Ci9reGlxWmdIL3ZKN016bHhlMy9UUFR2WXhUazFBbm9MR0JTOS9DNHBJb0c2bko1L1hxM3ZaVVhWQVZHeW5FaTgKRzZhRWxJc1ljNDBYTWpDKy9MQ0UzT214eGVoNkFWU0Nnei9lM1pIRXFWZjY1MGtUYnlmbEdqNEoyRXVUbW90QQpZemg5cVdSc2RiZFZ0S1BrdkxuZ2lSL3dxeEtKNjdJbndPbVQwV2hvTXViVHloZ1BJSm5Gc29pS3BrTlRPeWFPCkpHNXlEWFZuNlBzMHBZNWorSXBOMVFYTHl3SURBUUFCbzBJd1FEQU9CZ05WSFE4QkFmOEVCQU1DQWdRd0R3WUQKVlIwVEFRSC9CQVV3QXdFQi96QWRCZ05WSFE0RUZnUVUxYkExRFErTFZ2ZUxEeUVJSUZUeHBxRC9QbW93RFFZSgpLb1pJaHZjTkFRRUxCUUFEZ2dHQkFEazRnaDUrK3FNcjNVbEhDODNlTWJWbERmOEpBbHBzSkd0bEs1QzBjOVdSCk12bzRWMkFoNHpzeUpPS0pTNlRhd2RYMVlGQmxVUzJZNitkWTRBaDc0NXQwa29XWEs1R2JxNnJvNGROL3BteEEKZ2Q3N1JHNGFxZHUyK2NOYkRlOHkrSW9rSEJvN091MnJ3Zk1YeENWNSs0YWlJQUpaQlE5T3RyaVo5UFRBaFNiYwpJdUQ2TllxUTBrZE5Day9iZUhWZW01WCtiNGJ4aVJ6RUJZQkl4SHdac3VZT1NmR1lBVFE2QlZMOVJqQ0FKMi8zClNZS2JoSlJUQWxRN2kvYXhBWjZIZ1ROS25oeUNodk05UkZZV3BLVW5scEFMek9zR2RoU25LaU5PQXF0eW9JdWkKdVBzZHhNenloNGV1Z3cyeGtSQ2wxZ2ZxZXVJTnU1UjVWejk0Zm5LcFhleGFSd1FHdWlFejJ2bmpHTG9zV0QzTQpiUllkQjV1bEM5b1kzamc1c3RnTU5wTmw5Qyt3M3VLajd6WlhSQnBmNWFZMHpjUXZrdVJNdjFEYmNva3pFR2xiCjlzRVJBOGM5M3dmaERUK3lUUnZFSm5wNGY0YjFNcHlFN0RMSzZ0Z0VrVTduVXM3MDRBYXJzSUVUNTdpMDhqTlAKSjZZQzFDUURsQ0FXYTFudW14cFB5Zz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
+    server: https://34.67.1.138
+  name: gke_obot-infra_us-central1-a_obot
 contexts:
 - context:
     cluster: local-cluster
     user: local-user
   name: local-context
-current-context: local-context
+- context:
+    cluster: gke_obot-infra_us-central1-a_obot
+    user: gke_obot-infra_us-central1-a_obot
+  name: gke_obot-infra_us-central1-a_obot
+current-context: gke_obot-infra_us-central1-a_obot
 kind: Config
 preferences: {}
 users:
 - name: local-user
   user:
     token: adminpass
+- name: gke_obot-infra_us-central1-a_obot
+  user:
+    exec:
+      apiVersion: client.authentication.k8s.io/v1beta1
+      command: /opt/homebrew/share/google-cloud-sdk/bin/gke-gcloud-auth-plugin
+      installHint: Install gke-gcloud-auth-plugin for use with kubectl by following
+        https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-access-for-kubectl#install_plugin
+      provideClusterInfo: true