Direct images

SmartImage.Lib/Engines/BaseSearchEngine.cs

@@ -2,7 +2,9 @@
 using SmartImage.Lib.Searching;
 using System;
 using System.Diagnostics;
+using System.Linq;
 using System.Threading.Tasks;
+using SmartImage.Lib.Utilities;
 using static SimpleCore.Diagnostics.LogCategories;
 
 namespace SmartImage.Lib.Engines
@@ -20,6 +22,7 @@ protected BaseSearchEngine(string baseUrl)
 
 		public virtual string Name => Engine.ToString();
 
+
 		public virtual SearchResult GetResult(ImageQuery query)
 		{
 			var rawUrl = GetRawResultUrl(query);
@@ -34,6 +37,7 @@ public virtual SearchResult GetResult(ImageQuery query)
 				sr.Status = ResultStatus.Success;
 			}
 
+
 			return sr;
 		}
 
@@ -43,11 +47,11 @@ public async Task<SearchResult> GetResultAsync(ImageQuery query)
 
 			var task = Task.Run(delegate
 			{
-				Debug.WriteLine($"{Name}: getting result async",C_INFO);
-				
+				Debug.WriteLine($"{Name}: getting result async", C_INFO);
+
 				var res = GetResult(query);
-				
-				Debug.WriteLine($"{Name}: result done",C_SUCCESS);
+
+				Debug.WriteLine($"{Name}: result done", C_SUCCESS);
 
 				return res;
 			});
@@ -62,7 +66,7 @@ public Uri GetRawResultUrl(ImageQuery query)
 			bool ok = Network.IsUriAlive(uri);
 
 			if (!ok) {
-				Debug.WriteLine($"{uri.Host} is unavailable",C_WARN);
+				Debug.WriteLine($"{uri.Host} is unavailable", C_WARN);
 				return null;
 			}
 

SmartImage.Lib/Engines/Impl/Ascii2DEngine.cs

@@ -30,7 +30,7 @@ protected override IDocument GetDocument(SearchResult sr)
 		{
 			var url = sr.RawUri.ToString();
 
-			var res = Network.GetSimpleResponse(url);
+			var res = Network.GetResponse(url);
 
 			// Get redirect url (color url)
 			var newUrl = res.ResponseUri.ToString();

SmartImage.Lib/Engines/Impl/IqdbEngine.cs

@@ -5,12 +5,14 @@
 using System.Diagnostics;
 using System.Linq;
 using System.Threading;
+using System.Threading.Tasks;
 using AngleSharp.Dom;
 using AngleSharp.Html.Dom;
 using AngleSharp.XPath;
 using SimpleCore.Net;
 using SimpleCore.Utilities;
 using SmartImage.Lib.Searching;
+using SmartImage.Lib.Utilities;
 
 namespace SmartImage.Lib.Engines.Impl
 {
@@ -21,7 +23,7 @@ public IqdbEngine() : base("https://iqdb.org/?url=") { }
 		public override SearchEngineOptions Engine => SearchEngineOptions.Iqdb;
 
 		public override string Name => "IQDB";
-		
+
 
 		private static ImageResult ParseResult(IHtmlCollection<IElement> tr)
 		{
@@ -34,8 +36,7 @@ private static ImageResult ParseResult(IHtmlCollection<IElement> tr)
 			//img.ChildNodes[0].ChildNodes[0].TryGetAttribute("href")
 
 
-			try
-			{
+			try {
 				//url = src.FirstChild.ChildNodes[2].ChildNodes[0].TryGetAttribute("href");
 
 				url = img.ChildNodes[0].ChildNodes[0].TryGetAttribute("href");
@@ -79,7 +80,7 @@ private static ImageResult ParseResult(IHtmlCollection<IElement> tr)
 			}
 
 			var uri = url != null ? new Uri(url) : null;
-			
+
 
 			var result = new ImageResult
 			{
@@ -91,6 +92,10 @@ private static ImageResult ParseResult(IHtmlCollection<IElement> tr)
 				Description = caption.TextContent,
 			};
 
+
+
+
+
 			return result;
 		}
 

SmartImage.Lib/Engines/InterpretedSearchEngine.cs

@@ -25,6 +25,7 @@ public abstract class InterpretedSearchEngine : BaseSearchEngine
 
 		protected InterpretedSearchEngine(string baseUrl) : base(baseUrl) { }
 
+
 
 		[DebuggerHidden]
 		public override SearchResult GetResult(ImageQuery query)
@@ -60,7 +61,7 @@ protected virtual IDocument GetDocument(SearchResult sr)
 			}*/
 
 
-			string response = Network.GetString(sr.RawUri.ToString()!);
+			string response = WebUtilities.GetString(sr.RawUri.ToString()!);
 
 			var parser = new HtmlParser();
 			return parser.ParseDocument(response);

SmartImage.Lib/SearchClient.cs

@@ -130,15 +130,19 @@ public async Task RunSearchAsync()
 				var value = await finished;
 
 				tasks.Remove(finished);
-
+
+				if (Config.DirectUri) {
+					value.PrimaryResult.FindDirectImages();
+				}
+
 				if (!(Config.Filter && !value.IsNonPrimitive)) {
 					Results.Add(value);
-					
+
 					// Call event
 					ResultCompleted?.Invoke(null, new SearchResultEventArgs(value));
 				}
 
-				
+
 				IsComplete = !tasks.Any();
 			}
 

SmartImage.Lib/SearchConfig.cs

@@ -12,15 +12,33 @@ namespace SmartImage.Lib
 	/// <summary>
 	/// Contains configuration for <see cref="SearchClient"/>
 	/// </summary>
+	/// <remarks>Search config is only applicable when used in <see cref="SearchClient"/></remarks>
 	public sealed class SearchConfig
 	{
+		/// <summary>
+		/// Search query
+		/// </summary>
 		public ImageQuery Query { get; set; }
 
+		/// <summary>
+		/// Search engines to use
+		/// </summary>
 		public SearchEngineOptions SearchEngines { get; set; } = SearchEngineOptions.All;
 
+		/// <summary>
+		/// Priority engines
+		/// </summary>
 		public SearchEngineOptions PriorityEngines { get; set; }
 
+		/// <summary>
+		/// Filters any non-primitive results; <see cref="SearchResult.IsNonPrimitive"/>
+		/// </summary>
 		public bool Filter { get; set; } = true;
 
+		/// <summary>
+		/// Scan for direct image links; <see cref="ImageResult.FindDirectImages"/>
+		/// </summary>
+		public bool DirectUri { get; set; } = true;
+
 	}
 }

SmartImage.Lib/Searching/ImageQuery.cs

@@ -65,7 +65,7 @@ public ImageQuery([NotNull] string value, [CanBeNull] BaseUploadEngine engine =
 
 			Uri = IsUri ? new Uri(Value) : UploadEngine.Upload(Value);
 
-			Stream = IsFile ? File.OpenRead(value) : Network.GetStream(value);
+			Stream = IsFile ? File.OpenRead(value) : WebUtilities.GetStream(value);
 
 			Trace.WriteLine($"{nameof(ImageQuery)}: {Uri}", C_SUCCESS);
 		}

SmartImage.Lib/Searching/ImageResult.cs

@@ -3,11 +3,14 @@
 using SmartImage.Lib.Utilities;
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Drawing;
 using System.Linq;
 using System.Reflection;
 using System.Text;
+using System.Threading.Tasks;
 using Novus.Utilities;
+using static SimpleCore.Diagnostics.LogCategories;
 
 #nullable enable
 
@@ -19,10 +22,16 @@ namespace SmartImage.Lib.Searching
 	public sealed class ImageResult
 	{
 		/// <summary>
-		/// Url
+		/// Result url
 		/// </summary>
 		public Uri? Url { get; set; }
 
+		/// <summary>
+		/// Direct image link of <see cref="Url"/>
+		/// </summary>
+		public Uri? Direct { get; set; }
+
+
 		/// <summary>
 		/// Similarity
 		/// </summary>
@@ -167,6 +176,7 @@ public DisplayResolutionType DisplayResolution
 		public void UpdateFrom(ImageResult result)
 		{
 			Url         = result.Url;
+			Direct      = result.Direct;
 			Similarity  = result.Similarity;
 			Width       = result.Width;
 			Height      = result.Height;
@@ -178,12 +188,29 @@ public void UpdateFrom(ImageResult result)
 			Date        = result.Date;
 		}
 
+		public void FindDirectImages()
+		{
+
+				if (Url is not null) {
+					string? images = ImageHelper.FindDirectImages(Url?.ToString()).FirstOrDefault();
+
+					if (images is { }) {
+						var uri = new Uri(images);
+
+						Direct = uri;
+					}
+
+				}
+
+		}
+
 		public string ToString(bool indent)
 		{
 
 			var sb = new ExtendedStringBuilder() { };
 
 			sb.Append(nameof(Url), Url);
+			sb.Append(nameof(Direct), Direct);
 
 			if (Similarity.HasValue) {
 				sb.Append($"{nameof(Similarity)}", $"{Similarity.Value / 100:P}");

SmartImage.Lib/Utilities/ImageHelper.cs

@@ -13,6 +13,7 @@
 using AngleSharp.XPath;
 using Newtonsoft.Json.Linq;
 using SimpleCore.Net;
+using static SimpleCore.Diagnostics.LogCategories;
 using MimeType = SimpleCore.Net.MimeType;
 
 // ReSharper disable InconsistentNaming
@@ -43,8 +44,6 @@ public static class ImageHelper
 		 * https://github.com/regosen/gallery_get
 		 */
 
-
-
 
 		public static (int Width, int Height) GetResolution(string s)
 		{
@@ -88,54 +87,76 @@ public static DisplayResolutionType GetDisplayResolution(int w, int h)
 
 		}
 
-		public static bool IsDirect(string value) => MediaTypes.IsDirect(value, MimeType.Image);
+		/// <summary>
+		/// Determines whether <paramref name="url"/> is a direct image link
+		/// </summary>
+		/// <remarks>A direct image link is a link which points to a binary image file</remarks>
+		public static bool IsDirect(string url) => MediaTypes.IsDirect(url, MimeType.Image);
 
-		public static string[] Scan(string s)
+		/// <summary>
+		/// Scans for direct image links in <paramref name="url"/>
+		/// </summary>
+		public static string[] FindDirectImages(string url)
 		{
-
-			// TODO: WIP
+			var rg = new List<string>();
 
 			//<img.*?src="(.*?)"
 			//href\s*=\s*"(.+?)"
-
-			var html = Network.GetString(s);
-
 			//var src  = "<img.*?src=\"(.*?)\"";
 			//var href = "href\\s*=\\s*\"(.+?)\"";
-			var href = "<a\\s+(?:[^>]*?\\s+)?href=\"([^\"]*)\"";
-			//var m  = Regex.Matches(html, src);
-			var m2 = Regex.Matches(html, href);
 
-			//Debug.WriteLine($"{s} {m.Count} {m2.Count}");
+			var html = WebUtilities.GetString(url);
+
+			const string HREF_PATTERN = "<a\\s+(?:[^>]*?\\s+)?href=\"([^\"]*)\"";
 
+			var m2 = Regex.Matches(html, HREF_PATTERN);
 
-			for (int index = 0; index < m2.Count; index++) {
-				Match match = m2[index];
-				var   v     = match.Groups;
 
-				for (int i = 0; i < v.Count; i++) {
-					Group @group = v[i];
+			for (int i = 0; i < m2.Count; i++) {
+				var match  = m2[i];
+				var groups = match.Groups;
 
-					foreach (Capture capture in @group.Captures) {
-						// this works but it's slow
-						if (Network.IsUri(capture.Value, out var u)) {
-							Debug.WriteLine($"[{index}, {i}] {u}");
+				for (int j = 0; j < groups.Count; j++) {
+					var group = groups[j];
 
-						}
+					foreach (Capture capture in group.Captures) {
+
+						rg.Add(capture.Value);
 					}
 				}
 			}
 
+			string[] results = null;
+
+
+			var t = Task.Run(() =>
+			{
+				// todo: is running PLINQ within a task thread-safe?
+
+				results = rg.AsParallel()
+				            .Where(e => Network.IsUri(e, out _) && IsDirect(e))
+				            .ToArray();
+
+				Debug.WriteLine($"{nameof(FindDirectImages)}: {rg.Count} -> {results.Length}", C_DEBUG);
+			});
+
+
+			var timeout = TimeSpan.FromSeconds(3);
+
+			if (t.Wait(timeout)) {
+				//
+			}
+			else {
+				Debug.WriteLine($"{nameof(FindDirectImages)}: timeout!", C_WARN);
+			}
 
-			var rg = new List<string>();
 
-			return rg.ToArray();
+			return results;
 		}
 
 
 		/*public static string ResolveDirectLink(string s)
 		{
-			//todo: WIP
 			string d = "";
 
 			try {