diff --git a/go.mod b/go.mod index e71473ac..4347ea7b 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/gptscript-ai/gptscript go 1.22.0 -replace github.com/sashabaranov/go-openai => github.com/gptscript-ai/go-openai v0.0.0-20240206232711-45b6e096246a +replace github.com/sashabaranov/go-openai => github.com/gptscript-ai/go-openai v0.0.0-20240227161457-daa30caa3185 require ( github.com/BurntSushi/locker v0.0.0-20171006230638-a6e239ea1c69 diff --git a/go.sum b/go.sum index f55d6b1e..ab8fa286 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,8 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaU github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gptscript-ai/go-openai v0.0.0-20240206232711-45b6e096246a h1:AdBbQ1ODOYK5AwCey4VFEmKeu9gG4PCzuO80pQmgupE= -github.com/gptscript-ai/go-openai v0.0.0-20240206232711-45b6e096246a/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/gptscript-ai/go-openai v0.0.0-20240227161457-daa30caa3185 h1:+TfC9DYtWuexdL7x1lIdD1HP61IStb3ZTj/byBdiWs0= +github.com/gptscript-ai/go-openai v0.0.0-20240227161457-daa30caa3185/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/hexops/autogold v0.8.1/go.mod h1:97HLDXyG23akzAoRYJh/2OBs3kd80eHyKPvZw0S5ZBY= github.com/hexops/autogold v1.3.1 h1:YgxF9OHWbEIUjhDbpnLhgVsjUDsiHDTyDfy2lrfdlzo= github.com/hexops/autogold v1.3.1/go.mod h1:sQO+mQUCVfxOKPht+ipDSkJ2SCJ7BNJVHZexsXqWMx4= diff --git a/pkg/builtin/defaults.go b/pkg/builtin/defaults.go index 6b31ca90..ac264ae6 100644 --- a/pkg/builtin/defaults.go +++ b/pkg/builtin/defaults.go @@ -6,12 +6,20 @@ import ( ) var ( - DefaultModel = openai.DefaultModel + defaultModel = openai.DefaultModel ) +func GetDefaultModel() string { + return defaultModel +} + +func SetDefaultModel(model string) { + defaultModel = model +} + func SetDefaults(tool types.Tool) types.Tool { if tool.Parameters.ModelName == "" { - tool.Parameters.ModelName = DefaultModel + tool.Parameters.ModelName = GetDefaultModel() } return tool } diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 3c23673d..ab44fedf 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -1,6 +1,7 @@ package cache import ( + "context" "errors" "io/fs" "os" @@ -35,6 +36,17 @@ func complete(opts ...Options) (result Options) { return } +type noCacheKey struct{} + +func IsNoCache(ctx context.Context) bool { + v, _ := ctx.Value(noCacheKey{}).(bool) + return v +} + +func WithNoCache(ctx context.Context) context.Context { + return context.WithValue(ctx, noCacheKey{}, true) +} + func New(opts ...Options) (*Client, error) { opt := complete(opts...) if err := os.MkdirAll(opt.CacheDir, 0755); err != nil { diff --git a/pkg/cli/gptscript.go b/pkg/cli/gptscript.go index cdfbbfe7..bee280bd 100644 --- a/pkg/cli/gptscript.go +++ b/pkg/cli/gptscript.go @@ -10,8 +10,10 @@ import ( "github.com/acorn-io/cmd" "github.com/gptscript-ai/gptscript/pkg/assemble" "github.com/gptscript-ai/gptscript/pkg/builtin" + "github.com/gptscript-ai/gptscript/pkg/cache" "github.com/gptscript-ai/gptscript/pkg/engine" "github.com/gptscript-ai/gptscript/pkg/input" + "github.com/gptscript-ai/gptscript/pkg/llm" "github.com/gptscript-ai/gptscript/pkg/loader" "github.com/gptscript-ai/gptscript/pkg/monitor" "github.com/gptscript-ai/gptscript/pkg/mvl" @@ -26,10 +28,13 @@ import ( type ( DisplayOptions monitor.Options + CacheOptions cache.Options + OpenAIOptions openai.Options ) type GPTScript struct { - runner.Options + CacheOptions + OpenAIOptions DisplayOptions Debug bool `usage:"Enable debug logging"` Quiet *bool `usage:"No output logging" short:"q"` @@ -41,6 +46,8 @@ type GPTScript struct { ListTools bool `usage:"List built-in tools and exit"` Server bool `usage:"Start server"` ListenAddress string `usage:"Server listen address" default:"127.0.0.1:9090"` + + _client llm.Client `usage:"-"` } func New() *cobra.Command { @@ -67,6 +74,33 @@ func (r *GPTScript) Customize(cmd *cobra.Command) { } } +func (r *GPTScript) getClient(ctx context.Context) (llm.Client, error) { + if r._client != nil { + return r._client, nil + } + + cacheClient, err := cache.New(cache.Options(r.CacheOptions)) + if err != nil { + return nil, err + } + + oaClient, err := openai.NewClient(openai.Options(r.OpenAIOptions), openai.Options{ + Cache: cacheClient, + }) + if err != nil { + return nil, err + } + + registry := llm.NewRegistry() + + if err := registry.AddClient(ctx, oaClient); err != nil { + return nil, err + } + + r._client = registry + return r._client, nil +} + func (r *GPTScript) listTools() error { var lines []string for _, tool := range builtin.ListTools() { @@ -77,12 +111,12 @@ func (r *GPTScript) listTools() error { } func (r *GPTScript) listModels(ctx context.Context) error { - c, err := openai.NewClient(openai.Options(r.OpenAIOptions)) + c, err := r.getClient(ctx) if err != nil { return err } - models, err := c.ListModules(ctx) + models, err := c.ListModels(ctx) if err != nil { return err } @@ -95,6 +129,10 @@ func (r *GPTScript) listModels(ctx context.Context) error { } func (r *GPTScript) Pre(*cobra.Command, []string) error { + if r.DefaultModel != "" { + builtin.SetDefaultModel(r.DefaultModel) + } + if r.Quiet == nil { if term.IsTerminal(int(os.Stdout.Fd())) { r.Quiet = new(bool) @@ -126,9 +164,11 @@ func (r *GPTScript) Run(cmd *cobra.Command, args []string) error { } if r.Server { - s, err := server.New(server.Options{ - CacheOptions: r.CacheOptions, - OpenAIOptions: r.OpenAIOptions, + c, err := r.getClient(cmd.Context()) + if err != nil { + return err + } + s, err := server.New(c, server.Options{ ListenAddress: r.ListenAddress, }) if err != nil { @@ -176,9 +216,12 @@ func (r *GPTScript) Run(cmd *cobra.Command, args []string) error { return assemble.Assemble(prg, out) } - runner, err := runner.New(r.Options, runner.Options{ - CacheOptions: r.CacheOptions, - OpenAIOptions: r.OpenAIOptions, + client, err := r.getClient(cmd.Context()) + if err != nil { + return err + } + + runner, err := runner.New(client, runner.Options{ MonitorFactory: monitor.NewConsole(monitor.Options(r.DisplayOptions), monitor.Options{ DisplayProgress: !*r.Quiet, }), diff --git a/pkg/engine/cmd.go b/pkg/engine/cmd.go index fd89b412..1a06712a 100644 --- a/pkg/engine/cmd.go +++ b/pkg/engine/cmd.go @@ -12,7 +12,6 @@ import ( "sync/atomic" "github.com/google/shlex" - "github.com/gptscript-ai/gptscript/pkg/openai" "github.com/gptscript-ai/gptscript/pkg/types" "github.com/gptscript-ai/gptscript/pkg/version" ) @@ -21,7 +20,7 @@ func (e *Engine) runCommand(ctx context.Context, tool types.Tool, input string) id := fmt.Sprint(atomic.AddInt64(&completionID, 1)) defer func() { - e.Progress <- openai.Status{ + e.Progress <- types.CompletionStatus{ CompletionID: id, Response: map[string]any{ "output": cmdOut, @@ -31,7 +30,7 @@ func (e *Engine) runCommand(ctx context.Context, tool types.Tool, input string) }() if tool.BuiltinFunc != nil { - e.Progress <- openai.Status{ + e.Progress <- types.CompletionStatus{ CompletionID: id, Request: map[string]any{ "command": []string{tool.ID}, @@ -47,7 +46,7 @@ func (e *Engine) runCommand(ctx context.Context, tool types.Tool, input string) } defer stop() - e.Progress <- openai.Status{ + e.Progress <- types.CompletionStatus{ CompletionID: id, Request: map[string]any{ "command": cmd.Args, diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 3a5a37d9..83be656d 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -4,46 +4,16 @@ import ( "context" "encoding/json" "fmt" - "os" "sync" "sync/atomic" - "github.com/gptscript-ai/gptscript/pkg/openai" + "github.com/gptscript-ai/gptscript/pkg/system" "github.com/gptscript-ai/gptscript/pkg/types" "github.com/gptscript-ai/gptscript/pkg/version" ) -// InternalSystemPrompt is added to all threads. Changing this is very dangerous as it has a -// terrible global effect and changes the behavior of all scripts. -var InternalSystemPrompt = ` -You are task oriented system. -You receive input from a user, process the input from the given instructions, and then output the result. -Your objective is to provide consistent and correct results. -You do not need to explain the steps taken, only provide the result to the given instructions. -You are referred to as a tool. -` - -var DefaultToolSchema = types.JSONSchema{ - Property: types.Property{ - Type: "object", - }, - Properties: map[string]types.Property{ - openai.DefaultPromptParameter: { - Description: "Prompt to send to the tool or assistant. This may be instructions or question.", - Type: "string", - }, - }, - Required: []string{openai.DefaultPromptParameter}, -} - var completionID int64 -func init() { - if p := os.Getenv("GPTSCRIPT_INTERNAL_SYSTEM_PROMPT"); p != "" { - InternalSystemPrompt = p - } -} - type ErrToolNotFound struct { ToolName string } @@ -52,10 +22,14 @@ func (e *ErrToolNotFound) Error() string { return fmt.Sprintf("tool not found: %s", e.ToolName) } +type Model interface { + Call(ctx context.Context, messageRequest types.CompletionRequest, status chan<- types.CompletionStatus) (*types.CompletionMessage, error) +} + type Engine struct { - Client *openai.Client + Model Model Env []string - Progress chan<- openai.Status + Progress chan<- types.CompletionStatus } type State struct { @@ -172,18 +146,12 @@ func (e *Engine) Start(ctx Context, input string) (*Return, error) { } completion := types.CompletionRequest{ - Model: tool.Parameters.ModelName, - MaxToken: tool.Parameters.MaxTokens, - JSONResponse: tool.Parameters.JSONResponse, - Cache: tool.Parameters.Cache, - Temperature: tool.Parameters.Temperature, - } - - if InternalSystemPrompt != "" && (tool.Parameters.InternalPrompt == nil || *tool.Parameters.InternalPrompt) { - completion.Messages = append(completion.Messages, types.CompletionMessage{ - Role: types.CompletionMessageRoleTypeSystem, - Content: types.Text(InternalSystemPrompt), - }) + Model: tool.Parameters.ModelName, + MaxTokens: tool.Parameters.MaxTokens, + JSONResponse: tool.Parameters.JSONResponse, + Cache: tool.Parameters.Cache, + Temperature: tool.Parameters.Temperature, + InternalSystemPrompt: tool.Parameters.InternalPrompt, } for _, subToolName := range tool.Parameters.Tools { @@ -193,10 +161,9 @@ func (e *Engine) Start(ctx Context, input string) (*Return, error) { } args := subTool.Parameters.Arguments if args == nil && !subTool.IsCommand() { - args = &DefaultToolSchema + args = &system.DefaultToolSchema } completion.Tools = append(completion.Tools, types.CompletionTool{ - Type: types.CompletionToolTypeFunction, Function: types.CompletionFunctionDefinition{ Name: subToolName, Description: subTool.Parameters.Description, @@ -207,12 +174,8 @@ func (e *Engine) Start(ctx Context, input string) (*Return, error) { if tool.Instructions != "" { completion.Messages = append(completion.Messages, types.CompletionMessage{ - Role: types.CompletionMessageRoleTypeSystem, - Content: []types.ContentPart{ - { - Text: tool.Instructions, - }, - }, + Role: types.CompletionMessageRoleTypeSystem, + Content: types.Text(tool.Instructions), }) } @@ -230,7 +193,7 @@ func (e *Engine) Start(ctx Context, input string) (*Return, error) { func (e *Engine) complete(ctx context.Context, state *State) (*Return, error) { var ( - progress = make(chan openai.Status) + progress = make(chan types.CompletionStatus) ret = Return{ State: state, Calls: map[string]Call{}, @@ -241,6 +204,7 @@ func (e *Engine) complete(ctx context.Context, state *State) (*Return, error) { // ensure we aren't writing to the channel anymore on exit wg.Add(1) defer wg.Wait() + defer close(progress) go func() { defer wg.Done() @@ -251,8 +215,7 @@ func (e *Engine) complete(ctx context.Context, state *State) (*Return, error) { } }() - resp, err := e.Client.Call(ctx, state.Completion, progress) - close(progress) + resp, err := e.Model.Call(ctx, state.Completion, progress) if err != nil { return nil, err } diff --git a/pkg/hash/sha256.go b/pkg/hash/sha256.go index d941b152..be171281 100644 --- a/pkg/hash/sha256.go +++ b/pkg/hash/sha256.go @@ -6,6 +6,16 @@ import ( "encoding/json" ) +func Digest(obj any) string { + data, err := json.Marshal(obj) + if err != nil { + panic(err) + } + + hash := sha256.Sum224(data) + return hex.EncodeToString(hash[:]) +} + func Encode(obj any) string { data, err := json.Marshal(obj) if err != nil { diff --git a/pkg/llm/registry.go b/pkg/llm/registry.go new file mode 100644 index 00000000..20a3ae3b --- /dev/null +++ b/pkg/llm/registry.go @@ -0,0 +1,54 @@ +package llm + +import ( + "context" + "fmt" + "sort" + + "github.com/gptscript-ai/gptscript/pkg/types" +) + +type Client interface { + Call(ctx context.Context, messageRequest types.CompletionRequest, status chan<- types.CompletionStatus) (*types.CompletionMessage, error) + ListModels(ctx context.Context) (result []string, _ error) +} + +type Registry struct { + clientsByModel map[string]Client +} + +func NewRegistry() *Registry { + return &Registry{ + clientsByModel: map[string]Client{}, + } +} + +func (r *Registry) AddClient(ctx context.Context, client Client) error { + models, err := client.ListModels(ctx) + if err != nil { + return err + } + for _, model := range models { + r.clientsByModel[model] = client + } + return nil +} + +func (r *Registry) ListModels(_ context.Context) (result []string, _ error) { + for k := range r.clientsByModel { + result = append(result, k) + } + sort.Strings(result) + return result, nil +} + +func (r *Registry) Call(ctx context.Context, messageRequest types.CompletionRequest, status chan<- types.CompletionStatus) (*types.CompletionMessage, error) { + if messageRequest.Model == "" { + return nil, fmt.Errorf("model is required") + } + client, ok := r.clientsByModel[messageRequest.Model] + if !ok { + return nil, fmt.Errorf("model not found: %s", messageRequest.Model) + } + return client.Call(ctx, messageRequest, status) +} diff --git a/pkg/openai/client.go b/pkg/openai/client.go index 00d528e7..28f92680 100644 --- a/pkg/openai/client.go +++ b/pkg/openai/client.go @@ -15,13 +15,13 @@ import ( "github.com/gptscript-ai/gptscript/pkg/cache" "github.com/gptscript-ai/gptscript/pkg/hash" + "github.com/gptscript-ai/gptscript/pkg/system" "github.com/gptscript-ai/gptscript/pkg/types" "github.com/sashabaranov/go-openai" ) const ( - DefaultModel = openai.GPT4TurboPreview - DefaultPromptParameter = "defaultPromptParameter" + DefaultModel = openai.GPT4TurboPreview ) var ( @@ -31,19 +31,21 @@ var ( ) type Client struct { - url string - key string - c *openai.Client - cache *cache.Client + url string + key string + defaultModel string + c *openai.Client + cache *cache.Client } type Options struct { - BaseURL string `usage:"OpenAI base URL" name:"openai-base-url" env:"OPENAI_BASE_URL"` - APIKey string `usage:"OpenAI API KEY" name:"openai-api-key" env:"OPENAI_API_KEY"` - APIVersion string `usage:"OpenAI API Version (for Azure)" name:"openai-api-version" env:"OPENAI_API_VERSION"` - APIType openai.APIType `usage:"OpenAI API Type (valid: OPEN_AI, AZURE, AZURE_AD)" name:"openai-api-type" env:"OPENAI_API_TYPE"` - OrgID string `usage:"OpenAI organization ID" name:"openai-org-id" env:"OPENAI_ORG_ID"` - Cache *cache.Client + BaseURL string `usage:"OpenAI base URL" name:"openai-base-url" env:"OPENAI_BASE_URL"` + APIKey string `usage:"OpenAI API KEY" name:"openai-api-key" env:"OPENAI_API_KEY"` + APIVersion string `usage:"OpenAI API Version (for Azure)" name:"openai-api-version" env:"OPENAI_API_VERSION"` + APIType openai.APIType `usage:"OpenAI API Type (valid: OPEN_AI, AZURE, AZURE_AD)" name:"openai-api-type" env:"OPENAI_API_TYPE"` + OrgID string `usage:"OpenAI organization ID" name:"openai-org-id" env:"OPENAI_ORG_ID"` + DefaultModel string `usage:"Default LLM model to use" default:"gpt-4-turbo-preview"` + Cache *cache.Client } func complete(opts ...Options) (result Options, err error) { @@ -54,6 +56,7 @@ func complete(opts ...Options) (result Options, err error) { result.Cache = types.FirstSet(opt.Cache, result.Cache) result.APIVersion = types.FirstSet(opt.APIVersion, result.APIVersion) result.APIType = types.FirstSet(opt.APIType, result.APIType) + result.DefaultModel = types.FirstSet(opt.DefaultModel, result.DefaultModel) } if result.Cache == nil { @@ -70,7 +73,7 @@ func complete(opts ...Options) (result Options, err error) { result.APIKey = key } - if result.APIKey == "" { + if result.APIKey == "" && result.BaseURL == "" { return result, fmt.Errorf("OPENAI_API_KEY is not set. Please set the OPENAI_API_KEY environment variable") } @@ -94,12 +97,13 @@ func NewClient(opts ...Options) (*Client, error) { cfg.APIType = types.FirstSet(opt.APIType, cfg.APIType) return &Client{ - c: openai.NewClientWithConfig(cfg), - cache: opt.Cache, + c: openai.NewClientWithConfig(cfg), + cache: opt.Cache, + defaultModel: opt.DefaultModel, }, nil } -func (c *Client) ListModules(ctx context.Context) (result []string, _ error) { +func (c *Client) ListModels(ctx context.Context) (result []string, _ error) { models, err := c.c.ListModels(ctx) if err != nil { return nil, err @@ -138,7 +142,10 @@ func (c *Client) seed(request openai.ChatCompletionRequest) int { return hash.Seed(newRequest) } -func (c *Client) fromCache(messageRequest types.CompletionRequest, request openai.ChatCompletionRequest) (result []openai.ChatCompletionStreamResponse, _ bool, _ error) { +func (c *Client) fromCache(ctx context.Context, messageRequest types.CompletionRequest, request openai.ChatCompletionRequest) (result []openai.ChatCompletionStreamResponse, _ bool, _ error) { + if cache.IsNoCache(ctx) { + return nil, false, nil + } if messageRequest.Cache != nil && !*messageRequest.Cache { return nil, false, nil } @@ -161,7 +168,7 @@ func toToolCall(call types.CompletionToolCall) openai.ToolCall { return openai.ToolCall{ Index: call.Index, ID: call.ID, - Type: openai.ToolType(call.Type), + Type: openai.ToolTypeFunction, Function: openai.FunctionCall{ Name: call.Function.Name, Arguments: call.Function.Arguments, @@ -170,6 +177,13 @@ func toToolCall(call types.CompletionToolCall) openai.ToolCall { } func toMessages(request types.CompletionRequest) (result []openai.ChatCompletionMessage, err error) { + if request.InternalSystemPrompt == nil || *request.InternalSystemPrompt { + result = append(result, openai.ChatCompletionMessage{ + Role: openai.ChatMessageRoleSystem, + Content: system.InternalSystemPrompt, + }) + } + for _, message := range request.Messages { chatMessage := openai.ChatCompletionMessage{ Role: string(message.Role), @@ -198,13 +212,8 @@ func toMessages(request types.CompletionRequest) (result []openai.ChatCompletion chatMessage.Content = chatMessage.MultiContent[0].Text chatMessage.MultiContent = nil - if strings.Contains(chatMessage.Content, DefaultPromptParameter) && strings.HasPrefix(chatMessage.Content, "{") { - data := map[string]any{} - if err := json.Unmarshal([]byte(chatMessage.Content), &data); err == nil && len(data) == 1 { - if v, _ := data[DefaultPromptParameter].(string); v != "" { - chatMessage.Content = v - } - } + if prompt, ok := system.IsDefaultPrompt(chatMessage.Content); ok { + chatMessage.Content = prompt } } @@ -213,16 +222,10 @@ func toMessages(request types.CompletionRequest) (result []openai.ChatCompletion return } -type Status struct { - CompletionID string - Request any - Response any - Cached bool - Chunks any - PartialResponse *types.CompletionMessage -} - -func (c *Client) Call(ctx context.Context, messageRequest types.CompletionRequest, status chan<- Status) (*types.CompletionMessage, error) { +func (c *Client) Call(ctx context.Context, messageRequest types.CompletionRequest, status chan<- types.CompletionStatus) (*types.CompletionMessage, error) { + if messageRequest.Model == "" { + messageRequest.Model = c.defaultModel + } msgs, err := toMessages(messageRequest) if err != nil { return nil, err @@ -235,8 +238,9 @@ func (c *Client) Call(ctx context.Context, messageRequest types.CompletionReques request := openai.ChatCompletionRequest{ Model: messageRequest.Model, Messages: msgs, - MaxTokens: messageRequest.MaxToken, + MaxTokens: messageRequest.MaxTokens, Temperature: messageRequest.Temperature, + Grammar: messageRequest.Grammar, } if request.Temperature == nil { @@ -255,7 +259,7 @@ func (c *Client) Call(ctx context.Context, messageRequest types.CompletionReques params.Properties = map[string]types.Property{} } request.Tools = append(request.Tools, openai.Tool{ - Type: openai.ToolType(tool.Type), + Type: openai.ToolTypeFunction, Function: openai.FunctionDefinition{ Name: tool.Function.Name, Description: tool.Function.Description, @@ -265,14 +269,14 @@ func (c *Client) Call(ctx context.Context, messageRequest types.CompletionReques } id := fmt.Sprint(atomic.AddInt64(&completionID, 1)) - status <- Status{ + status <- types.CompletionStatus{ CompletionID: id, Request: request, } var cacheResponse bool request.Seed = ptr(c.seed(request)) - response, ok, err := c.fromCache(messageRequest, request) + response, ok, err := c.fromCache(ctx, messageRequest, request) if err != nil { return nil, err } else if !ok { @@ -289,7 +293,7 @@ func (c *Client) Call(ctx context.Context, messageRequest types.CompletionReques result = appendMessage(result, response) } - status <- Status{ + status <- types.CompletionStatus{ CompletionID: id, Chunks: response, Response: result, @@ -328,7 +332,6 @@ func appendMessage(msg types.CompletionMessage, response openai.ChatCompletionSt tc.ToolCall.Index = tool.Index } tc.ToolCall.ID = override(tc.ToolCall.ID, tool.ID) - tc.ToolCall.Type = types.CompletionToolType(override(string(tc.ToolCall.Type), string(tool.Type))) tc.ToolCall.Function.Name += tool.Function.Name tc.ToolCall.Function.Arguments += tool.Function.Arguments @@ -364,7 +367,10 @@ func override(left, right string) string { return left } -func (c *Client) store(key string, responses []openai.ChatCompletionStreamResponse) error { +func (c *Client) store(ctx context.Context, key string, responses []openai.ChatCompletionStreamResponse) error { + if cache.IsNoCache(ctx) { + return nil + } buf := &bytes.Buffer{} gz := gzip.NewWriter(buf) err := json.NewEncoder(gz).Encode(responses) @@ -377,11 +383,11 @@ func (c *Client) store(key string, responses []openai.ChatCompletionStreamRespon return c.cache.Store(key, buf.Bytes()) } -func (c *Client) call(ctx context.Context, request openai.ChatCompletionRequest, transactionID string, partial chan<- Status) (responses []openai.ChatCompletionStreamResponse, _ error) { +func (c *Client) call(ctx context.Context, request openai.ChatCompletionRequest, transactionID string, partial chan<- types.CompletionStatus) (responses []openai.ChatCompletionStreamResponse, _ error) { cacheKey := c.cacheKey(request) request.Stream = true - partial <- Status{ + partial <- types.CompletionStatus{ CompletionID: transactionID, PartialResponse: &types.CompletionMessage{ Role: types.CompletionMessageRoleTypeAssistant, @@ -400,14 +406,14 @@ func (c *Client) call(ctx context.Context, request openai.ChatCompletionRequest, for { response, err := stream.Recv() if err == io.EOF { - return responses, c.store(cacheKey, responses) + return responses, c.store(ctx, cacheKey, responses) } else if err != nil { return nil, err } slog.Debug("stream", "content", response.Choices[0].Delta.Content) if partial != nil { partialMessage = appendMessage(partialMessage, response) - partial <- Status{ + partial <- types.CompletionStatus{ CompletionID: transactionID, PartialResponse: &partialMessage, } diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index d277ff4f..7c55ccba 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -5,9 +5,7 @@ import ( "sync" "time" - "github.com/gptscript-ai/gptscript/pkg/cache" "github.com/gptscript-ai/gptscript/pkg/engine" - "github.com/gptscript-ai/gptscript/pkg/openai" "github.com/gptscript-ai/gptscript/pkg/types" "golang.org/x/sync/errgroup" ) @@ -21,21 +19,12 @@ type Monitor interface { Stop(output string, err error) } -type ( - CacheOptions cache.Options - OpenAIOptions openai.Options -) - type Options struct { - CacheOptions - OpenAIOptions MonitorFactory MonitorFactory `usage:"-"` } -func complete(opts ...Options) (cacheOpts []cache.Options, oaOpts []openai.Options, result Options) { +func complete(opts ...Options) (result Options) { for _, opt := range opts { - cacheOpts = append(cacheOpts, cache.Options(opt.CacheOptions)) - oaOpts = append(oaOpts, openai.Options(opt.OpenAIOptions)) result.MonitorFactory = types.FirstSet(opt.MonitorFactory, result.MonitorFactory) } if result.MonitorFactory == nil { @@ -45,26 +34,15 @@ func complete(opts ...Options) (cacheOpts []cache.Options, oaOpts []openai.Optio } type Runner struct { - c *openai.Client + c engine.Model factory MonitorFactory } -func New(opts ...Options) (*Runner, error) { - cacheOpts, oaOpts, opt := complete(opts...) - cacheClient, err := cache.New(cacheOpts...) - if err != nil { - return nil, err - } - - oaClient, err := openai.NewClient(append(oaOpts, openai.Options{ - Cache: cacheClient, - })...) - if err != nil { - return nil, err - } +func New(client engine.Model, opts ...Options) (*Runner, error) { + opt := complete(opts...) return &Runner{ - c: oaClient, + c: client, factory: opt.MonitorFactory, }, nil } @@ -111,7 +89,7 @@ func (r *Runner) call(callCtx engine.Context, monitor Monitor, env []string, inp defer progressClose() e := engine.Engine{ - Client: r.c, + Model: r.c, Progress: progress, Env: env, } @@ -166,8 +144,8 @@ func (r *Runner) call(callCtx engine.Context, monitor Monitor, env []string, inp } } -func streamProgress(callCtx *engine.Context, monitor Monitor) (chan openai.Status, func()) { - progress := make(chan openai.Status) +func streamProgress(callCtx *engine.Context, monitor Monitor) (chan types.CompletionStatus, func()) { + progress := make(chan types.CompletionStatus) wg := sync.WaitGroup{} wg.Add(1) diff --git a/pkg/server/server.go b/pkg/server/server.go index 60ee1c2c..802164f0 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -16,6 +16,8 @@ import ( "github.com/acorn-io/broadcaster" "github.com/gptscript-ai/gptscript/pkg/builtin" + "github.com/gptscript-ai/gptscript/pkg/cache" + "github.com/gptscript-ai/gptscript/pkg/engine" "github.com/gptscript-ai/gptscript/pkg/loader" "github.com/gptscript-ai/gptscript/pkg/runner" "github.com/gptscript-ai/gptscript/pkg/types" @@ -25,18 +27,12 @@ import ( ) type Options struct { - runner.CacheOptions - runner.OpenAIOptions ListenAddress string } -func complete(opts []Options) (runnerOpts []runner.Options, result Options) { +func complete(opts []Options) (result Options) { for _, opt := range opts { result.ListenAddress = types.FirstSet(opt.ListenAddress, result.ListenAddress) - runnerOpts = append(runnerOpts, runner.Options{ - CacheOptions: opt.CacheOptions, - OpenAIOptions: opt.OpenAIOptions, - }) } if result.ListenAddress == "" { result.ListenAddress = "127.0.0.1:9090" @@ -44,27 +40,15 @@ func complete(opts []Options) (runnerOpts []runner.Options, result Options) { return } -func New(opts ...Options) (*Server, error) { +func New(model engine.Model, opts ...Options) (*Server, error) { events := broadcaster.New[Event]() - runnerOpts, opt := complete(opts) - r, err := runner.New(append(runnerOpts, runner.Options{ + opt := complete(opts) + r, err := runner.New(model, runner.Options{ MonitorFactory: &SessionFactory{ events: events, }, - })...) - if err != nil { - return nil, err - } - - noCacheRunner, err := runner.New(append(runnerOpts, runner.Options{ - CacheOptions: runner.CacheOptions{ - Cache: new(bool), - }, - MonitorFactory: &SessionFactory{ - events: events, - }, - })...) + }) if err != nil { return nil, err } @@ -73,7 +57,6 @@ func New(opts ...Options) (*Server, error) { melody: melody.New(), events: events, runner: r, - noCacheRunner: noCacheRunner, listenAddress: opt.ListenAddress, }, nil } @@ -91,7 +74,6 @@ type Server struct { ctx context.Context melody *melody.Melody runner *runner.Runner - noCacheRunner *runner.Runner events *broadcaster.Broadcaster[Event] listenAddress string } @@ -168,16 +150,10 @@ func (s *Server) run(rw http.ResponseWriter, req *http.Request) { return } - runner := s.runner - if req.URL.Query().Has("nocache") { - runner = s.noCacheRunner - } - - id := fmt.Sprint(atomic.AddInt64(&execID, 1)) - if req.URL.Query().Has("async") { - ctx := context.WithValue(s.ctx, execKey{}, id) + id, ctx := s.getContext(req) + if isAsync(req) { go func() { - _, _ = runner.Run(ctx, prg, os.Environ(), string(body)) + _, _ = s.runner.Run(ctx, prg, os.Environ(), string(body)) }() rw.Header().Set("Content-Type", "application/json") err := json.NewEncoder(rw).Encode(map[string]any{ @@ -187,8 +163,7 @@ func (s *Server) run(rw http.ResponseWriter, req *http.Request) { http.Error(rw, err.Error(), http.StatusInternalServerError) } } else { - ctx := context.WithValue(req.Context(), execKey{}, id) - out, err := runner.Run(ctx, prg, os.Environ(), string(body)) + out, err := s.runner.Run(ctx, prg, os.Environ(), string(body)) if err == nil { _, _ = rw.Write([]byte(out)) } else { @@ -197,6 +172,24 @@ func (s *Server) run(rw http.ResponseWriter, req *http.Request) { } } +func isAsync(req *http.Request) bool { + return req.URL.Query().Has("async") +} + +func (s *Server) getContext(req *http.Request) (string, context.Context) { + ctx := req.Context() + if req.URL.Query().Has("async") { + ctx = s.ctx + } + + id := fmt.Sprint(atomic.AddInt64(&execID, 1)) + ctx = context.WithValue(ctx, execKey{}, id) + if req.URL.Query().Has("nocache") { + ctx = cache.WithNoCache(ctx) + } + return id, ctx +} + func (s *Server) Start(ctx context.Context) error { s.ctx = ctx s.melody.HandleConnect(s.Connect) diff --git a/pkg/system/prompt.go b/pkg/system/prompt.go new file mode 100644 index 00000000..4de97b85 --- /dev/null +++ b/pkg/system/prompt.go @@ -0,0 +1,56 @@ +package system + +import ( + "encoding/json" + "os" + "strings" + + "github.com/gptscript-ai/gptscript/pkg/types" +) + +// InternalSystemPrompt is added to all threads. Changing this is very dangerous as it has a +// terrible global effect and changes the behavior of all scripts. +var InternalSystemPrompt = ` +You are task oriented system. +You receive input from a user, process the input from the given instructions, and then output the result. +Your objective is to provide consistent and correct results. +You do not need to explain the steps taken, only provide the result to the given instructions. +You are referred to as a tool. +` + +// DefaultPromptParameter is used as the key in a json map to indication that we really wanted +// to just send pure text but the interface required JSON (as that is the fundamental interface of tools in OpenAI) +var DefaultPromptParameter = "defaultPromptParameter" + +var DefaultToolSchema = types.JSONSchema{ + Property: types.Property{ + Type: "object", + }, + Properties: map[string]types.Property{ + DefaultPromptParameter: { + Description: "Prompt to send to the tool or assistant. This may be instructions or question.", + Type: "string", + }, + }, + Required: []string{DefaultPromptParameter}, +} + +func init() { + if p := os.Getenv("GPTSCRIPT_INTERNAL_SYSTEM_PROMPT"); p != "" { + InternalSystemPrompt = p + } +} + +// IsDefaultPrompt Checks if the content is a json blob that has the defaultPromptParameter in it. If so +// it will extract out the value and return it. If not it will return the original content as is and false. +func IsDefaultPrompt(content string) (string, bool) { + if strings.Contains(content, DefaultPromptParameter) && strings.HasPrefix(content, "{") { + data := map[string]any{} + if err := json.Unmarshal([]byte(content), &data); err == nil && len(data) == 1 { + if v, _ := data[DefaultPromptParameter].(string); v != "" { + return v, true + } + } + } + return content, false +} diff --git a/pkg/test/examples_test.go b/pkg/test/examples_test.go index 2a02507a..bc0e3e85 100644 --- a/pkg/test/examples_test.go +++ b/pkg/test/examples_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/gptscript-ai/gptscript/pkg/loader" + "github.com/gptscript-ai/gptscript/pkg/openai" "github.com/gptscript-ai/gptscript/pkg/runner" "github.com/hexops/autogold/v2" "github.com/stretchr/testify/require" @@ -23,9 +24,13 @@ func TestExamples(t *testing.T) { "fac.gpt", "helloworld.gpt", } + + client, err := openai.NewClient() + require.NoError(t, err) + for _, entry := range tests { t.Run(entry, func(t *testing.T) { - r, err := runner.New() + r, err := runner.New(client) require.NoError(t, err) prg, err := loader.Program(context.Background(), filepath.Join(examplePath, entry), "") @@ -42,7 +47,10 @@ func TestExamples(t *testing.T) { func TestEcho(t *testing.T) { RequireOpenAPIKey(t) - r, err := runner.New() + client, err := openai.NewClient() + require.NoError(t, err) + + r, err := runner.New(client) require.NoError(t, err) prg, err := loader.Program(context.Background(), filepath.Join(examplePath, "echo.gpt"), "") diff --git a/pkg/types/completion.go b/pkg/types/completion.go index 90f6db08..7d7d927e 100644 --- a/pkg/types/completion.go +++ b/pkg/types/completion.go @@ -5,24 +5,19 @@ import ( "strings" ) -const ( - CompletionToolTypeFunction CompletionToolType = "function" -) - -type CompletionToolType string - type CompletionRequest struct { - Model string - Tools []CompletionTool - Messages []CompletionMessage - MaxToken int - Temperature *float32 - JSONResponse bool - Cache *bool + Model string + InternalSystemPrompt *bool + Tools []CompletionTool + Messages []CompletionMessage + MaxTokens int + Temperature *float32 + JSONResponse bool + Grammar string + Cache *bool } type CompletionTool struct { - Type CompletionToolType `json:"type"` Function CompletionFunctionDefinition `json:"function,omitempty"` } @@ -44,9 +39,20 @@ const ( type CompletionMessageRoleType string type CompletionMessage struct { - Role CompletionMessageRoleType `json:"role,omitempty"` - Content []ContentPart `json:"content,omitempty" column:"name=Message,jsonpath=.spec.content"` - ToolCall *CompletionToolCall `json:"toolCall,omitempty"` + Role CompletionMessageRoleType `json:"role,omitempty"` + Content []ContentPart `json:"content,omitempty" column:"name=Message,jsonpath=.spec.content"` + // ToolCall should be set for only messages of type "tool" and Content[0].Text should be set as the + // result of the call describe by this field + ToolCall *CompletionToolCall `json:"toolCall,omitempty"` +} + +type CompletionStatus struct { + CompletionID string + Request any + Response any + Cached bool + Chunks any + PartialResponse *CompletionMessage } func (in CompletionMessage) IsToolCall() bool { @@ -88,7 +94,6 @@ type ContentPart struct { type CompletionToolCall struct { Index *int `json:"index,omitempty"` ID string `json:"id,omitempty"` - Type CompletionToolType `json:"type,omitempty"` Function CompletionFunctionCall `json:"function,omitempty"` }