Working FTP

ConfigNames.cs

@@ -8,6 +8,8 @@ public static class ConfigNames
 
     public const string PortFtp = "portftp";
 
+    public const string PortSocks5 = "portsocks5";
+
     public const string InternetArchive = "internetarchive";
 
     public const string InternetArchiveYear = "internetarchiveyear";

Data/Cache/CacheDbContext.cs

@@ -101,7 +101,7 @@ internal Tuple<uint, uint> GetCounters()
         {
             var proxyCacheCountCommand = context.CreateCommand();
 
-            proxyCacheCountCommand.CommandText = "SELECT COUNT(*) FROM cache WHERE key LIKE 'PC-%'";
+            proxyCacheCountCommand.CommandText = "SELECT COUNT(*) FROM cache WHERE key LIKE '%PC-%'";
 
             using var proxyCacheCountReader = proxyCacheCountCommand.ExecuteReader();
 

Data/Config/ConfigDbContext.cs

@@ -10,6 +10,7 @@ internal class ConfigDbContext : DbContextBase, IConfigDb
         { ConfigNames.IpAddress, IPAddress.Any.ToString() },
         { ConfigNames.PortHttp, 1990 },
         { ConfigNames.PortFtp, 1971 },
+        { ConfigNames.PortSocks5, 1996 },
         { ConfigNames.Intranet, true },
         { ConfigNames.ProtoWeb, true },
         { ConfigNames.InternetArchive, true },

Mind.cs

@@ -1,11 +1,11 @@
 using System.Net;
-using VintageHive.Data;
 using VintageHive.Data.Cache;
 using VintageHive.Data.Config;
 using VintageHive.Processors;
 using VintageHive.Proxy.Ftp;
 using VintageHive.Proxy.Http;
 using VintageHive.Proxy.Security;
+using VintageHive.Proxy.Socks;
 using VintageHive.Utilities;
 
 namespace VintageHive;
@@ -24,10 +24,12 @@ class Mind
 
     HttpProxy _httpProxy;
 
-    HttpProxy _httpsProxy;
+    // HttpProxy _httpsProxy;
 
     FtpProxy _ftpProxy;
 
+    //Socks5Proxy _socks5Proxy;
+
     public IConfigDb ConfigDb => _configDb;
 
     public ICacheDb CacheDb => _cacheDb;
@@ -62,31 +64,36 @@ public async Task Init()
 
         await CheckGeoIp();
 
-        // TODO: Ugh, fix this later.
-        if (ConfigDb.SettingGet<bool>(ConfigNames.ProtoWeb))
-        {
-            ProtoWebProcessor.AvailableSites = await ProtoWebUtils.GetAvailableSites();
-        }
-
         var ipAddressString = ConfigDb.SettingGet<string>(ConfigNames.IpAddress);
 
         var ipAddress = IPAddress.Parse(ipAddressString);
 
-        var httpPort = ConfigDb.SettingGet<int>(ConfigNames.PortHttp);        
-
-        _httpProxy = new(ipAddress, httpPort, false);
+        var httpPort = ConfigDb.SettingGet<int>(ConfigNames.PortHttp);
 
-        _httpProxy.CacheDb = _cacheDb;
+        _httpProxy = new(ipAddress, httpPort, false)
+        {
+            CacheDb = _cacheDb
+        };
 
         _httpProxy
             .Use(IntranetProcessor.ProcessRequest)
             .Use(RedirectionHelper.ProcessRequest)
-            .Use(ProtoWebProcessor.ProcessRequest)
+            .Use(ProtoWebProcessor.ProcessHttpRequest)
             .Use(InternetArchiveProcessor.ProcessRequest);
 
         var ftpPort = ConfigDb.SettingGet<int>(ConfigNames.PortFtp);
 
-        _ftpProxy = new(ipAddress, ftpPort);
+        _ftpProxy = new(ipAddress, ftpPort)
+        {
+            CacheDb = _cacheDb
+        };
+
+        _ftpProxy
+            .Use(ProtoWebProcessor.ProcessFtpRequest);
+
+        //var socks5Port = ConfigDb.SettingGet<int>(ConfigNames.PortSocks5);
+
+        //_socks5Proxy = new(ipAddress, socks5Port);
 
         // _httpsProxy = new(ipAddress, 9999, true);
 
@@ -142,6 +149,8 @@ internal void Start()
 
         _ftpProxy.Start();
 
+        //_socks5Proxy.Start();
+
         _resetEvent.WaitOne();
     }
 }

Processors/InternetArchiveProcessor.cs

@@ -103,12 +103,14 @@ public static async Task<bool> ProcessRequest(HttpRequest req, HttpResponse res)
 
             var iaResponse = await httpClient.GetAsync(iaUrl);
 
+            httpClient.Dispose();
+
             if (iaResponse.StatusCode == System.Net.HttpStatusCode.NotFound)
             {
                 return false;
             }
 
-            contentType = iaResponse.Content.Headers.ContentType.ToString() ?? "text/html";
+            contentType = iaResponse.Content.Headers.ContentType?.ToString() ?? "text/html";
 
             if (contentType.StartsWith("text/html"))
             {

Processors/ProtoWebProcessor.cs

@@ -1,14 +1,26 @@
 using System.Diagnostics;
+using System.Net.Sockets;
+using System.Text;
+using VintageHive.Proxy.Ftp;
 using VintageHive.Proxy.Http;
 using VintageHive.Utilities;
+using System;
 
 namespace VintageHive.Processors;
 
 internal static class ProtoWebProcessor
 {
-    public static List<string> AvailableSites;
+    const string FetchRequestUserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.3705)";
+
+    const string RedirectPacketSignature = "HTTP/1.0 301 Moved Permanently\n";
 
-    public static async Task<bool> ProcessRequest(HttpRequest req, HttpResponse res)
+    static byte[] RedirectPacketSignatureBytes { get; } = Encoding.ASCII.GetBytes(RedirectPacketSignature);
+
+    static List<string> AvailableHttpSites;
+
+    static List<string> AvailableFtpSites;
+
+    public static async Task<bool> ProcessHttpRequest(HttpRequest req, HttpResponse res)
     {
         var protoWebEnabled = Mind.Instance.ConfigDb.SettingGet<bool>(ConfigNames.ProtoWeb);
 
@@ -17,17 +29,17 @@ public static async Task<bool> ProcessRequest(HttpRequest req, HttpResponse res)
             return false;
         }
 
-        if (AvailableSites == null)
+        if (AvailableHttpSites == null)
         {
-            AvailableSites = await ProtoWebUtils.GetAvailableSites();
+            AvailableHttpSites = await ProtoWebUtils.GetAvailableHttpSites();
 
-            if (AvailableSites == null)
+            if (AvailableHttpSites == null)
             {                
                 return false;
             }
         }
 
-        if (AvailableSites.Any(x => req.Uri.Host.EndsWith(x)))
+        if (AvailableHttpSites.Any(x => req.Uri.Host.EndsWith(x)))
         {
             var proxyClient = Clients.GetProxiedHttpClient(req, ProtoWebUtils.MainProxyUri);
 
@@ -41,7 +53,7 @@ public static async Task<bool> ProcessRequest(HttpRequest req, HttpResponse res)
 
                 res.CacheTtl = TimeSpan.FromDays(30);
 
-                Console.WriteLine($"[{"ProtoWeb", 15} Request] ({req.Uri}) [{contentType}]");
+                Console.WriteLine($"[{"ProtoWeb HTTP", 17} Request] ({req.Uri}) [{contentType}]");
 
                 return true;
             }
@@ -56,4 +68,74 @@ public static async Task<bool> ProcessRequest(HttpRequest req, HttpResponse res)
 
         return false;
     }
+
+    public static async Task<byte[]> ProcessFtpRequest(FtpRequest req)
+    {
+        if (AvailableFtpSites == null)
+        {
+            AvailableFtpSites = await ProtoWebUtils.GetAvailableFtpSites();
+
+            if (AvailableFtpSites == null)
+            {
+                return null;
+            }
+        }
+
+        if (AvailableFtpSites.Any(x => req.Uri.Host.EndsWith(x)))
+        {
+            Console.WriteLine($"[{"ProtoWeb  FTP", 17} Request] ({req.Uri})");
+
+            // We're streaming the data to the client, so it's not caching.
+            // We could cache the small pages, but file downloads are not appropriate for SQLite storage.
+            // We could possibly cache the files to disk...
+            if (req.Uri.Scheme != "ftp")
+            {
+                return null;
+            }
+
+            using var tcpClient = new TcpClient(ProtoWebUtils.MainProxyHost, ProtoWebUtils.MainProxyPort)
+            {
+                NoDelay = true,
+            };
+
+            tcpClient.Client.NoDelay = true;
+
+            var stream = tcpClient.GetStream();
+
+            var request = ProtoWebUtils.CreateFtpContentRequest(req.Uri.ToString(), FetchRequestUserAgent, req.Uri.Host);
+
+            var rawRequest = Encoding.ASCII.GetBytes(request);
+
+            stream.Write(rawRequest, 0, rawRequest.Length);
+
+            var buffer = new byte[1024];
+
+            var readBytes = await stream.ReadAsync(buffer);
+
+            if (readBytes == RedirectPacketSignatureBytes.Length && buffer.Take(RedirectPacketSignatureBytes.Length).SequenceEqual(RedirectPacketSignatureBytes))
+            {
+                readBytes = await stream.ReadAsync(buffer);
+
+                var responseText = Encoding.ASCII.GetString(buffer, 0, readBytes);
+
+                var location = responseText.Split("Location: ")[1].Split("\n")[0];
+
+                req.Uri = new Uri(location);
+
+                await ProcessFtpRequest(req);
+
+                return null;
+            }
+
+            var clientStream = new NetworkStream(req.ListenerSocket.RawSocket);
+
+            await clientStream.WriteAsync(buffer.Take(readBytes).ToArray());
+
+            await stream.CopyToAsync(clientStream);
+
+            return null;
+        }
+
+        return null;
+    }
 }

Proxy/Ftp/FtpProxy.cs

@@ -1,30 +1,91 @@
 using System.Net;
 using System.Net.Sockets;
 using System.Text;
+using VintageHive.Data.Cache;
+using VintageHiveFtpProcessDelegate = System.Func<VintageHive.Proxy.Ftp.FtpRequest, System.Threading.Tasks.Task<byte[]>>;
 
 namespace VintageHive.Proxy.Ftp;
 
 public class FtpProxy : Listener
 {
-    const string NewLine = "\r\n";
+    static TimeSpan CacheTtl { get; set; } = TimeSpan.FromDays(7);
+
+    readonly List<VintageHiveFtpProcessDelegate> Handlers = new();
+
+    internal ICacheDb CacheDb { get; set; }
 
     public FtpProxy(IPAddress listenAddress, int port) : base(listenAddress, port, SocketType.Stream, ProtocolType.Tcp) { }
 
+    public FtpProxy Use(VintageHiveFtpProcessDelegate delegateFunc)
+    {
+        Handlers.Add(delegateFunc);
+
+        return this;
+    }
+
     internal override async Task<byte[]> ProcessConnection(ListenerSocket connection)
     {
         await Task.Delay(0);
-        
-        return Encoding.ASCII.GetBytes("VintageHive FTP Proxy" + NewLine);
+
+        return Encoding.ASCII.GetBytes("220 VintageHive FTP Proxy!\n");
     }
 
     internal override async Task<byte[]> ProcessRequest(ListenerSocket connection, byte[] data, int read)
     {
         var requestData = Encoding.ASCII.GetString(data, 0, read);
 
-        // System.Diagnostics.Debugger.Break();
+        var req = FtpRequest.ParseFtpOverHttp(connection, Encoding, data);
+
+        var key = $"FPC-{req.Uri}";
+
+        var cachedResponse = CacheDb?.Get<string>(key);
+
+        if (cachedResponse == null)
+        {
+            byte[] responseData = null;
+
+            try
+            {
+                foreach (var handler in Handlers)
+                {
+                    responseData = await handler(req);
+
+                    if (responseData != null)
+                    {
+                        break;
+                    }
+                }
+            }
+            catch (Exception)
+            {
+                return null;
+            }
+
+            if (connection == null)
+            {
+                // Nothing to send too..
+                return null;
+            }
+
+            if (responseData != null)
+            {
+                CacheDb?.Set<string>(key, CacheTtl, Convert.ToBase64String(responseData));
+            }
+
+            return responseData;
+        }
+        else
+        {
+            try
+            {
+                Console.WriteLine($"[{"FTP Proxy Cached",15} Request] ({req.Uri}) [N/A]");
 
-        await connection.RawSocket.SendAsync(Encoding.ASCII.GetBytes("FolderA\\" + NewLine + "FolderB\\" + NewLine), SocketFlags.None);
+                return Convert.FromBase64String(cachedResponse);
+            }
+            catch (Exception) { }
+        }
 
+        // TODO: Keep-Alive respect?
         connection.RawSocket.Close();
 
         return null;