Skip to content

Commit

Permalink
Work on #64, #63, #62 ...
Browse files Browse the repository at this point in the history 
Refactor SearchClient & P/C model

Normalize args; fixes
  • Loading branch information
@Decimation
Decimation committed Jan 2, 2025
1 parent 07d2d09 commit 94ab5fd
Show file tree
Hide file tree
Showing 15 changed files with 249 additions and 162 deletions.
2 changes: 1 addition & 1 deletion SmartImage.Lib/Images/Uni/UniImage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
using SixLabors.ImageSharp.Processing;
using SmartImage.Lib.Results.Data;

#pragma warning disable CS0168 // Variable is declared but never used
// #pragma warning disable CS0168 // Variable is declared but never used

namespace SmartImage.Lib.Images.Uni;
#nullable disable
Expand Down
2 changes: 1 addition & 1 deletion SmartImage.Lib/Results/Data/ISimilarity.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

namespace SmartImage.Lib.Results.Data;

#pragma warning disable CS0168
// #pragma warning disable CS0168
public interface ISimilarity
{

Expand Down
218 changes: 112 additions & 106 deletions SmartImage.Lib/SearchClient.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
global using CMN = System.Runtime.CompilerServices.CallerMemberNameAttribute;
global using EC = System.Runtime.CompilerServices.EnumeratorCancellationAttribute;
global using CMN = System.Runtime.CompilerServices.CallerMemberNameAttribute;
global using JI = System.Text.Json.Serialization.JsonIgnoreAttribute;
global using ICBN = JetBrains.Annotations.ItemCanBeNullAttribute;
global using INN = JetBrains.Annotations.ItemNotNullAttribute;
Expand Down Expand Up @@ -34,12 +35,15 @@
using SmartImage.Lib.Results.Data;
using static System.Runtime.InteropServices.JavaScript.JSType;
using SmartImage.Lib.Utilities.Diagnostics;
using Kantan.Monad;

namespace SmartImage.Lib;

public sealed class SearchClient : IDisposable
{

public SearchQuery Query { get; set; }

public SearchConfig Config { get; init; }

public bool IsComplete { get; private set; }
Expand All @@ -52,22 +56,38 @@ public sealed class SearchClient : IDisposable

private static readonly ILogger s_logger = AppSupport.Factory.CreateLogger(nameof(SearchClient));

public SearchClient(SearchConfig cfg)
private static readonly Lock m_lock = new Lock();

public SearchClient(SearchConfig cfg, SearchQuery query)
{
Query = query;
Config = cfg;
ConfigApplied = false;
IsRunning = false;

Config.PropertyChanged += (sender, args) =>
{
if (args.PropertyName == nameof(SearchConfig.SearchEngines)) {
lock (m_lock) {
Engines = BaseSearchEngine.GetSelectedEngines(Config.SearchEngines).ToArray();

}
}
};

// GetSelectedEngines();

}

static SearchClient()
{ }
public SearchClient(SearchConfig cfg) : this(cfg, SearchQuery.Null) { }

static SearchClient() { }

[ModuleInitializer]
public static void Init()
{
Trace.AutoFlush = true;
Debug.AutoFlush = true;
s_logger.LogInformation("Init");


Expand Down Expand Up @@ -99,100 +119,71 @@ public void OpenChannel()
{
var ok = ResultChannel?.Writer.TryComplete(new ChannelClosedException("Reopened channel"));

if (ok.HasValue && ok.Value) { }
if (ok.HasValue && ok.Value) {
// ...
}

ResultChannel = Channel.CreateUnbounded<SearchResult>(new UnboundedChannelOptions()
ResultChannel = Channel.CreateBounded<SearchResult>(new BoundedChannelOptions(Engines.Length)
{
SingleWriter = true,
});
}

/// <summary>
/// Runs a search of <paramref name="query"/>.
/// </summary>
/// <param name="query">Search query</param>
/// <param name="scheduler"></param>
/// <param name="token">Cancellation token passed to <see cref="WebSearchEngine{T}.GetResultAsync(SmartImage.Lib.SearchQuery,System.Threading.CancellationToken)"/></param>
public async Task<SearchResult[]> RunSearchAsync(SearchQuery query,
TaskScheduler scheduler = default,
CancellationToken token = default)
public async IAsyncEnumerable<SearchResult> RunSearchAsync(TaskScheduler scheduler = default,
[EC] CancellationToken token = default)
{
scheduler ??= TaskScheduler.Default;
await RunSearchAsync2(token);

// Requires.NotNull(ResultChannel);
if (ResultChannel == null || (IsComplete && !IsRunning)) {
OpenChannel();
}

if (!query.IsUploaded) {
throw new ArgumentException($"Query was not uploaded", nameof(query));
while (await ResultChannel.Reader.WaitToReadAsync(token)) {
while (ResultChannel.Reader.TryRead(out var result)) {
yield return result;
}
}
}

IsRunning = true;

if (!ConfigApplied) {
await LoadEnginesAsync(token); // todo
public ValueTask<bool> RunSearchAsync2(CancellationToken token = default)
=> RunSearchAsync2(ResultChannel.Writer, token);

}
else {
Debug.WriteLine("Not reloading engines");
public async ValueTask<bool> RunSearchAsync2(ChannelWriter<SearchResult> cw,
CancellationToken token = default)
{
if (!Query.IsUploaded) {
throw new SmartImageException($"{Query} was not uploaded");
}

Debug.WriteLine($"Config: {Config} | {Engines.QuickJoin()}");
IEnumerable<Task<SearchResult>> tasks;

List<Task<SearchResult>> tasks = GetSearchTasks(query, scheduler, token).ToList();
lock (m_lock) {
tasks = Engines.Select(e =>
{
var task = e.GetResultAsync(Query, token);
return task;
});
}

var results = new SearchResult[tasks.Count];
int i = 0;
await foreach (var task in Task.WhenEach(tasks).WithCancellation(token)) {

while (tasks.Count > 0) {
if (token.IsCancellationRequested) {
var result = await task;

Debugger.Break();
s_logger.LogWarning("Cancellation requested");
CompleteSearchAsync();
return results;
if (task.IsFaulted || task.IsCanceled) {
Trace.WriteLine($"{task} faulted or was canceled");
}

Task<SearchResult> task = await Task.WhenAny(tasks);
tasks.Remove(task);

if (task.IsFaulted) {
Trace.WriteLine($"{task} faulted!",LogCategories.C_ERROR);
if (cw.TryWrite(result)) {
//
}
SearchResult result = await task;

results[i] = result;
i++;
}

CompleteSearchAsync();
OnSearchComplete?.Invoke(this, results);

if (Config.PriorityEngines == SearchEngineOptions.Auto) {
if (Config.PriorityEngines.HasFlag(result.Engine.EngineOption)) {
var url = Config.OpenRaw ? result.RawUrl : result.GetBestResult()?.Url;

try {

SearchResultItem item = GetBest(results);

if (item != null) {
OpenResult(item.Url);
}
}
catch (Exception e) {
Debug.WriteLine($"{e.Message}");

Debugger.Break();
OpenResult(url);
}

}

IsRunning = false;

return results;
return default;
}

public static SearchResultItem GetBest(SearchResult[] results)
public static SearchResultItem GetBest(IEnumerable<SearchResult> results)
{
var ordered = results.Select(x => x.GetBestResult())
.Where(x => x != null)
Expand Down Expand Up @@ -259,16 +250,10 @@ public IEnumerable<Task<SearchResult>> GetSearchTasks(SearchQuery query, TaskSch
{
var tasks = Engines.Select(e =>
{
try {
/*try {
Debug.WriteLine($"Starting {e} for {query}");
Task<SearchResult> res = e.GetResultAsync(query, token: token)
.ContinueWith((r) =>
{
ProcessResult(r.Result);
return r.Result;
}, token, TaskContinuationOptions.None, scheduler);
return res;
}
Expand All @@ -279,7 +264,19 @@ public IEnumerable<Task<SearchResult>> GetSearchTasks(SearchQuery query, TaskSch
// return Task.FromException(exception);
}
return default;
return default;*/

/*Task<SearchResult> res = e.GetResultAsync(query, token: token)
.ContinueWith((r) =>
{
ProcessResult(r.Result);
return r.Result;
}, token, TaskContinuationOptions.None, scheduler)*/
;

Task<SearchResult> res = e.GetResultAsync(query, token: token);
return res;
});

return tasks;
Expand All @@ -291,40 +288,16 @@ public async ValueTask LoadEnginesAsync(CancellationToken token = default)

Trace.WriteLine("Loading engines");

Engines = BaseSearchEngine.GetSelectedEngines(Config.SearchEngines).ToArray();

if (Config.ReadCookies) {

try {
await ((DefaultCookiesProvider) DefaultCookiesProvider.Instance).OpenAsync();
}
catch (Exception e) {
Trace.WriteLine($"{e}");
Config.ReadCookies = false;
DefaultCookiesProvider.Instance.Dispose();
}
await InitCookiesProvider();
}

if (Config.FlareSolverr && !FlareSolverrClient.Value.IsInitialized) {


var ok = FlareSolverrClient.Value.Configure(Config.FlareSolverrApiUrl);

if (!ok) {
Debugger.Break();
}
else {
// Ensure FlareSolverr

try {
var idx = await FlareSolverrClient.Value.Clearance.Solverr.GetIndexAsync();
}
catch (Exception e) {
Trace.WriteLine($"{nameof(FlareSolverrClient)}: {e.Message}");
Config.FlareSolverr = false;
FlareSolverrClient.Value.Dispose();
}
}
await InitFlareSolverr();
}

foreach (BaseSearchEngine bse in Engines) {
Expand All @@ -346,6 +319,39 @@ public async ValueTask LoadEnginesAsync(CancellationToken token = default)
ConfigApplied = true;
}

private async Task InitFlareSolverr()
{
var ok = FlareSolverrClient.Value.Configure(Config.FlareSolverrApiUrl);

if (!ok) {
Debugger.Break();
}
else {
// Ensure FlareSolverr

try {
var idx = await FlareSolverrClient.Value.Clearance.Solverr.GetIndexAsync();
}
catch (Exception e) {
Trace.WriteLine($"{nameof(FlareSolverrClient)}: {e.Message}");
Config.FlareSolverr = false;
FlareSolverrClient.Value.Dispose();
}
}
}

private async Task InitCookiesProvider()
{
try {
await ((DefaultCookiesProvider) DefaultCookiesProvider.Instance).OpenAsync();
}
catch (Exception e) {
Trace.WriteLine($"{e}");
Config.ReadCookies = false;
DefaultCookiesProvider.Instance.Dispose();
}
}

public void Dispose()
{
Debug.WriteLine($"Disposing {nameof(SearchClient)}");
Expand All @@ -360,4 +366,4 @@ public void Dispose()
ResultChannel?.Writer.Complete();
}

}
}
13 changes: 9 additions & 4 deletions SmartImage.Lib/SearchServer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,14 +74,19 @@ private async Task<object> HandleRequestAsync(HttpListenerRequest request, HttpL
Debug.WriteLine($"{sz}");

var sq = await SearchQuery.TryCreateAsync(sz);

Client.Query = sq;
if (sq == SearchQuery.Null) {
return R1.Err_Query;
}

var url = await sq.UploadAsync();

var results = await Client.RunSearchAsync(sq);
// todo
var results1 = Client.RunSearchAsync();
var results = new List<SearchResult>();
await foreach (var v in results1) {
results.Add(v);
}

var best = SearchClient.GetBest(results);

Expand Down Expand Up @@ -125,9 +130,9 @@ public class SearchResults
public SearchResultItem Best { get; internal set; }

[JsonPropertyOrder(1)]
public SearchResult[] Results { get; }
public IList<SearchResult> Results { get; }

public SearchResults(SearchResult[] results, SearchResultItem best)
public SearchResults(IList<SearchResult> results, SearchResultItem best)
{
Best = best;
Results = results;
Expand Down
2 changes: 1 addition & 1 deletion SmartImage.Lib/SmartImage.Lib.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
<PublishTrimmed>True</PublishTrimmed>
<!-- <PublishSingleFile>true</PublishSingleFile> -->
<PublishSingleFile Condition="'$(Configuration)' == 'Release|AnyCPU'">true</PublishSingleFile>
<NoWarn>AD0001;IDE0290;CS0168</NoWarn>
<NoWarn>AD0001;IDE0290</NoWarn>
</PropertyGroup>
<PropertyGroup>
<SatelliteResourceLanguages>en-US;en</SatelliteResourceLanguages>
Expand Down
Loading

0 comments on commit 94ab5fd

Please sign in to comment.