From 0dc8bb42087d7f0530def3abcab459443a2a33b7 Mon Sep 17 00:00:00 2001 From: Horst Beham Date: Sun, 28 Jan 2024 04:04:01 +0100 Subject: [PATCH] - changed geo-ip client to use https://ip-api.com/docs/api:batch --- ServerBrowser/GeoIpClient.cs | 154 ++++++++---------- ServerBrowser/Program.cs | 4 +- ServerBrowser/ServerBrowserForm.cs | 14 +- .../ServerSources/MasterServerClient.cs | 2 +- ServerBrowser/XWebClient.cs | 4 + 5 files changed, 82 insertions(+), 96 deletions(-) diff --git a/ServerBrowser/GeoIpClient.cs b/ServerBrowser/GeoIpClient.cs index a163eac..75913b8 100644 --- a/ServerBrowser/GeoIpClient.cs +++ b/ServerBrowser/GeoIpClient.cs @@ -11,11 +11,12 @@ namespace ServerBrowser { + /// + /// Geo-IP client using the free https://ip-api.com/docs/api:batch service + /// class GeoIpClient { - internal const int ThreadCount = 7; - private const string DefaultServiceUrlFormat = "http://api.ipapi.com/{0}?access_key=9c5fc4375488ed26aa2f26b613324f4a&language=en&output=json"; - private DateTime usageExceeded = DateTime.MinValue; + private const string DefaultServiceUrlFormat = "http://ip-api.com/batch?fields=223&lang=en"; /// /// the cache holds either a GeoInfo object, or a multicast callback delegate waiting for a GeoInfo object @@ -30,8 +31,7 @@ public GeoIpClient(string cacheFile) { this.cacheFile = cacheFile; this.ServiceUrlFormat = DefaultServiceUrlFormat; - for (int i=0; i this.ProcessLoop()); + ThreadPool.QueueUserWorkItem(context => this.ProcessLoop()); } #region ProcessLoop() @@ -39,39 +39,61 @@ private void ProcessLoop() { using (var client = new XWebClient(5000)) { + int sleepMillis = 1000; while (true) { - var ip = this.queue.Take(); - if (ip == null) + Thread.Sleep(sleepMillis); + var count = Math.Max(1, Math.Min(100, this.queue.Count)); + var ips = new IPAddress[count]; + for (int i = 0; i < count; i++) + ips[i] = this.queue.Take(); + if (ips[ips.Length - 1] == null) break; - bool err = true; - var ipInt = Ip4Utils.ToInt(ip); + var req = new StringBuilder(count * 18); + req.Append("["); + foreach (var ip in ips) + req.Append('"').Append(ip).Append("\","); + req[req.Length - 1] = ']'; + + Dictionary geoInfos = null; try { - var url = string.Format(this.ServiceUrlFormat, ip); - var result = client.DownloadString(url); - if (result != null) - { - object o; - Action callbacks; - lock (cache) - callbacks = cache.TryGetValue(ipInt, out o) ? o as Action : null; - var geoInfo = this.HandleResult(ipInt, result); - if (callbacks != null && geoInfo != null) - ThreadPool.QueueUserWorkItem(ctx => callbacks(geoInfo)); - err = false; - } + var result = client.UploadString(ServiceUrlFormat, req.ToString()); + var rateLimit = client.ResponseHeaders["X-Rl"]; + sleepMillis = rateLimit == "0" ? (int.TryParse(client.ResponseHeaders["X-Ttl"], out var sec) ? sec * 1000: 4000) : 0; + geoInfos = this.HandleResult(ips, result); } catch { // ignore } - if (err) - { - lock (this.cache) - this.cache.Remove(ipInt); + foreach (var ip in ips) + { + var ipInt = Ip4Utils.ToInt(ip); + Action callbacks = null; + GeoInfo geoInfo = null; + lock (cache) + { + bool isSet = cache.TryGetValue(ipInt, out var o); + if (geoInfos == null || !geoInfos.TryGetValue(ipInt, out geoInfo)) + { + //this.cache.Remove(ipInt); + } + else + { + callbacks = o as Action; + if (geoInfo != null || !isSet) + cache[ipInt] = geoInfo; + } + } + + if (callbacks != null && geoInfo != null) + { + //ThreadPool.QueueUserWorkItem(ctx => callbacks(geoInfo)); + callbacks(geoInfo); + } } } } @@ -79,26 +101,22 @@ private void ProcessLoop() #endregion #region HandleResult() - private GeoInfo HandleResult(uint ip, string result) + private Dictionary HandleResult(IList ips, string result) { - var ser = new DataContractJsonSerializer(typeof(NekudoGeopIpFullResponse)); - var info = (NekudoGeopIpFullResponse)ser.ReadObject(new MemoryStream(Encoding.UTF8.GetBytes(result))); + var ser = new DataContractJsonSerializer(typeof(IpApiResponse[])); + var infoArray = (IpApiResponse[])ser.ReadObject(new MemoryStream(Encoding.UTF8.GetBytes(result))); - if (!info.success) + var ok = ips.Count == infoArray.Length; + var data = new Dictionary(); + for (int i = 0; i < ips.Count; i++) { - if (info.error?.code == 104) - this.usageExceeded = DateTime.UtcNow.Date; - lock (cache) - this.cache.Remove(ip); - return null; + var ipInt = Ip4Utils.ToInt(ips[i]); + var info = infoArray[i]; + var geoInfo = ok ? new GeoInfo(info.countryCode, info.country, info.region, info.regionName, info.city, info.lat, info.lon) : null; + data[ipInt] = geoInfo; } - var geoInfo = new GeoInfo(info.country_code, info.country_name, info.region_code, info.region_name, info.city, info.latitude, info.longitude); - lock (cache) - { - cache[ip] = geoInfo; - } - return geoInfo; + return data; } #endregion @@ -122,9 +140,6 @@ public void Lookup(IPAddress ip, Action callback) if (geoInfo == null) { - if (this.usageExceeded == DateTime.UtcNow.Date) - return; - if (cached == null) { cache.Add(ipInt, callback); @@ -272,51 +287,16 @@ public override string ToString() } #endregion -#if false - private class NekudoGeopIpShortResponse - { - public class Country - { - public string name; - public string code; - } - - public class Location - { - public decimal latitude; - public decimal longitude; - public string time_zone; - } - - public string city; - public Country country; - public Location location; - public string ip; - } -#endif - - #region class NekudoGeopIpFullResponse - public class NekudoGeopIpFullResponse + #region class IpApiResponse + public class IpApiResponse { - public class NekudoGeoIpError - { - public int code; - public string type; - public string info; - } - - public bool success; - public NekudoGeoIpError error; - - public string ip; - public string country_code; - public string country_name; - public string region_code; - public string region_name; + public string country; + public string countryCode; + public string region; + public string regionName; public string city; - public decimal latitude; - public decimal longitude; + public decimal lat; + public decimal lon; } #endregion - } diff --git a/ServerBrowser/Program.cs b/ServerBrowser/Program.cs index 57b6e39..1a6dee1 100644 --- a/ServerBrowser/Program.cs +++ b/ServerBrowser/Program.cs @@ -41,8 +41,8 @@ public static void Init(string fontName, float fontSize, string skinName) Application.EnableVisualStyles(); Application.SetCompatibleTextRenderingDefault(false); - ThreadPool.SetMinThreads(50 + GeoIpClient.ThreadCount + 5, 100); - ThreadPool.SetMaxThreads(50 + GeoIpClient.ThreadCount + 5, 100); + ThreadPool.SetMinThreads(50 + 1 + 5, 100); + ThreadPool.SetMaxThreads(50 + 1 + 5, 100); AppearanceObject.DefaultFont = new Font(fontName, fontSize); // must not create a Font instance before initializing GDI+ UserLookAndFeel.Default.SkinName = skinName; diff --git a/ServerBrowser/ServerBrowserForm.cs b/ServerBrowser/ServerBrowserForm.cs index f8e0777..8fab244 100644 --- a/ServerBrowser/ServerBrowserForm.cs +++ b/ServerBrowser/ServerBrowserForm.cs @@ -33,7 +33,7 @@ namespace ServerBrowser { public partial class ServerBrowserForm : XtraForm { - private const string Version = "2.65"; + private const string Version = "2.66"; private const string DevExpressVersion = "v23.2"; private const string OldSteamWebApiText = ""; private const string CustomDetailColumnPrefix = "ServerInfo."; @@ -703,13 +703,14 @@ protected virtual void UpdateViewModel() #region CreateServerSource() protected virtual IServerSource CreateServerSource(string addressAndPort) { - var endpoint = Ip4Utils.ParseEndpoint(addressAndPort); string steamWebApiKey = ""; - if (endpoint.Port == 0) + IPEndPoint endpoint = null; + if (Regex.IsMatch(addressAndPort, @"^[\dA-Fa-f]{32}$")) + steamWebApiKey = addressAndPort; + else { - if (Regex.IsMatch(addressAndPort, @"^[\dA-Fa-f]{32}$")) - steamWebApiKey = addressAndPort; - else + endpoint = Ip4Utils.ParseEndpoint(addressAndPort); + if (endpoint.Port == 0) { var result = XtraMessageBox.Show(this, "To use the \"\" as a Master Server, you need a Steam Web API Key (32 hex digits).\n" + "Valve issues these keys free of charge to web site owners and developers, but not necessarily to users.\n" + @@ -727,6 +728,7 @@ protected virtual IServerSource CreateServerSource(string addressAndPort) return null; } } + return new MasterServerClient(endpoint, steamWebApiKey); } #endregion diff --git a/ServerBrowser/ServerSources/MasterServerClient.cs b/ServerBrowser/ServerSources/MasterServerClient.cs index c9a5403..da22db9 100644 --- a/ServerBrowser/ServerSources/MasterServerClient.cs +++ b/ServerBrowser/ServerSources/MasterServerClient.cs @@ -16,7 +16,7 @@ public MasterServerClient(IPEndPoint masterServerEndpoint, string steamWebApiKey public void GetAddresses(Region region, IpFilter filter, int maxResults, MasterIpCallback callback) { - var master = masterServerEndPoint.Port == 0 ? (MasterServer)new MasterServerWebApi(steamWebApiKey) : new MasterServerUdp(masterServerEndPoint); + var master = masterServerEndPoint?.Port == null ? (MasterServer)new MasterServerWebApi(steamWebApiKey) : new MasterServerUdp(masterServerEndPoint); master.GetAddressesLimit = maxResults; master.GetAddresses(region, callback, filter); } diff --git a/ServerBrowser/XWebClient.cs b/ServerBrowser/XWebClient.cs index 0459d52..f9ee982 100644 --- a/ServerBrowser/XWebClient.cs +++ b/ServerBrowser/XWebClient.cs @@ -50,6 +50,7 @@ public void Dispose() #endregion public WebHeaderCollection Headers { get; set; } = new WebHeaderCollection(); + public WebHeaderCollection ResponseHeaders { get; private set; } public int ResumeOffset { get; set; } @@ -159,6 +160,8 @@ public byte[] DownloadData(Uri url) var req = this.CreateWebRequest(url); using (var res = req.GetResponse()) { + ResponseHeaders = res.Headers; + var mem = new MemoryStream((int)req.ContentLength); using (var r = res.GetResponseStream()) { @@ -220,6 +223,7 @@ public string UploadString(Uri url, string data) using (var res = req.GetResponse()) using (var r = new StreamReader(res.GetResponseStream(), this.Encoding)) { + ResponseHeaders = res.Headers; return r.ReadToEnd(); } }