-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
35 changed files
with
1,457 additions
and
391 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| using System.Text.Json.Serialization; | ||
|
|
||
| namespace SpongeEngine.KoboldSharp | ||
| { | ||
| public partial class KoboldSharpClient | ||
| { | ||
| public class KoboldSharpRequest | ||
| { | ||
| [JsonPropertyName("prompt")] | ||
| public string Prompt { get; set; } = string.Empty; | ||
|
|
||
| [JsonPropertyName("max_length")] | ||
| public int MaxLength { get; set; } = 80; | ||
|
|
||
| [JsonPropertyName("max_context_length")] | ||
| public int? MaxContextLength { get; set; } | ||
|
|
||
| [JsonPropertyName("temperature")] | ||
| public float Temperature { get; set; } = 0.7f; | ||
|
|
||
| [JsonPropertyName("top_p")] | ||
| public float TopP { get; set; } = 0.9f; | ||
|
|
||
| [JsonPropertyName("top_k")] | ||
| public int TopK { get; set; } = 40; | ||
|
|
||
| [JsonPropertyName("top_a")] | ||
| public float TopA { get; set; } = 0.0f; | ||
|
|
||
| [JsonPropertyName("typical")] | ||
| public float Typical { get; set; } = 1.0f; | ||
|
|
||
| [JsonPropertyName("tfs")] | ||
| public float Tfs { get; set; } = 1.0f; | ||
|
|
||
| [JsonPropertyName("rep_pen")] | ||
| public float RepetitionPenalty { get; set; } = 1.1f; | ||
|
|
||
| [JsonPropertyName("rep_pen_range")] | ||
| public int RepetitionPenaltyRange { get; set; } = 64; | ||
|
|
||
| [JsonPropertyName("mirostat")] | ||
| public int MirostatMode { get; set; } = 0; | ||
|
|
||
| [JsonPropertyName("mirostat_tau")] | ||
| public float MirostatTau { get; set; } = 5.0f; | ||
|
|
||
| [JsonPropertyName("mirostat_eta")] | ||
| public float MirostatEta { get; set; } = 0.1f; | ||
|
|
||
| [JsonPropertyName("stop_sequence")] | ||
| public List<string>? StopSequences { get; set; } | ||
|
|
||
| [JsonPropertyName("stream")] | ||
| public bool Stream { get; set; } | ||
|
|
||
| [JsonPropertyName("trim_stop")] | ||
| public bool TrimStop { get; set; } = true; | ||
|
|
||
| [JsonPropertyName("grammar")] | ||
| public string? Grammar { get; set; } | ||
|
|
||
| [JsonPropertyName("memory")] | ||
| public string? Memory { get; set; } | ||
|
|
||
| [JsonPropertyName("banned_tokens")] | ||
| public List<string>? BannedTokens { get; set; } | ||
|
|
||
| [JsonPropertyName("logit_bias")] | ||
| public Dictionary<string, float>? LogitBias { get; set; } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| using System.Net.Http.Json; | ||
| using System.Text.Json; | ||
| using System.Text.Json.Serialization; | ||
| using SpongeEngine.LLMSharp.Core.Exceptions; | ||
|
|
||
| namespace SpongeEngine.KoboldSharp | ||
| { | ||
| public partial class KoboldSharpClient | ||
| { | ||
| private class AbortResponse | ||
| { | ||
| [JsonPropertyName("success")] | ||
| public string Success { get; set; } = string.Empty; | ||
|
|
||
| [JsonPropertyName("done")] | ||
| public string Done { get; set; } = string.Empty; | ||
| } | ||
|
|
||
| public async Task<bool> AbortGenerateAsync(string? genKey = null, CancellationToken cancellationToken = default) | ||
| { | ||
| try | ||
| { | ||
| using HttpRequestMessage httpRequest = new(HttpMethod.Post, "api/extra/abort"); | ||
| if (!string.IsNullOrEmpty(genKey)) | ||
| { | ||
| httpRequest.Content = JsonContent.Create(new { genkey = genKey }); | ||
| } | ||
|
|
||
| using HttpResponseMessage? response = await Options.HttpClient.SendAsync(httpRequest, cancellationToken); | ||
| string? content = await response.Content.ReadAsStringAsync(cancellationToken); | ||
|
|
||
| if (!response.IsSuccessStatusCode) | ||
| { | ||
| throw new LlmSharpException( | ||
| "Failed to abort generation", | ||
| (int)response.StatusCode, | ||
| content); | ||
| } | ||
|
|
||
| var result = JsonSerializer.Deserialize<AbortResponse>(content); | ||
| return result?.Success == "true"; | ||
| } | ||
| catch (Exception ex) when (ex is not LlmSharpException) | ||
| { | ||
| throw new LlmSharpException("Failed to abort generation", innerException: ex); | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| using System.Net.Http.Json; | ||
| using System.Text.Json; | ||
| using System.Text.Json.Serialization; | ||
| using SpongeEngine.LLMSharp.Core.Exceptions; | ||
|
|
||
| namespace SpongeEngine.KoboldSharp | ||
| { | ||
| public partial class KoboldSharpClient | ||
| { | ||
| public class CountTokensRequest | ||
| { | ||
| [JsonPropertyName("prompt")] | ||
| public string Prompt { get; set; } = string.Empty; | ||
|
|
||
| [JsonPropertyName("special")] | ||
| public bool IncludeSpecialTokens { get; set; } = true; | ||
| } | ||
|
|
||
| public class CountTokensResponse | ||
| { | ||
| [JsonPropertyName("value")] | ||
| public int Count { get; set; } | ||
|
|
||
| [JsonPropertyName("ids")] | ||
| public List<int> TokenIds { get; set; } = new(); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Counts tokens in a given prompt string. | ||
| /// </summary> | ||
| public async Task<CountTokensResponse> CountTokensAsync(CountTokensRequest request, CancellationToken cancellationToken = default) | ||
| { | ||
| try | ||
| { | ||
| using HttpRequestMessage httpRequest = new(HttpMethod.Post, "api/extra/tokencount"); | ||
| httpRequest.Content = JsonContent.Create(request); | ||
|
|
||
| using HttpResponseMessage? response = await Options.HttpClient.SendAsync(httpRequest, cancellationToken); | ||
| string? content = await response.Content.ReadAsStringAsync(cancellationToken); | ||
|
|
||
| if (!response.IsSuccessStatusCode) | ||
| { | ||
| throw new LlmSharpException( | ||
| "Failed to count tokens", | ||
| (int)response.StatusCode, | ||
| content); | ||
| } | ||
|
|
||
| return JsonSerializer.Deserialize<CountTokensResponse>(content) ?? throw new LlmSharpException("Failed to deserialize token count response"); | ||
| } | ||
| catch (Exception ex) when (ex is not LlmSharpException) | ||
| { | ||
| throw new LlmSharpException("Failed to count tokens", innerException: ex); | ||
| } | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.