Merge pull request #83 from ngld/pr/fix-ftp-listing

Replace FTP file listing with HTTP file listing
scp-fs2open · Apr 17, 2023 · f222b6c · f222b6c
2 parents 1b20a24 + 90f7f61
commit f222b6c
Show file tree

Hide file tree

Showing 6 changed files with 70 additions and 63 deletions.
diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml
@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.8]
 
     steps:
       - uses: actions/checkout@v2

diff --git a/config.yml.sample b/config.yml.sample
@@ -38,10 +38,6 @@ templates:
     installer: templates/installer.mako
 
 ftp:
-    host: example.com
-    user: asdf
-    pass: xyz
-    path: public_html/builds/{type}/{version}/
     mirrors:
       - http://scp.indiegames.us/builds/{type}/{version}/{file}
       - https://porphyrion.feralhosting.com/datacorder/builds/{type}/{version}/{file}

diff --git a/file_list.py b/file_list.py
@@ -1,5 +1,4 @@
 import re
-from ftplib import FTP, error_perm
 from typing import List, Tuple, Dict
 
 import requests
@@ -57,24 +56,29 @@ def execute_request(path):
     return binary_files, source_files
 
 
-def get_ftp_files(build_type, tag_name, config):
+def get_nightly_files(tag_name, config):
     tag_regex = re.compile("nightly_(.*)")
     build_group_regex = re.compile("nightly_.*-builds-([^.]+).*")
+    link_regex = re.compile(r'<a href="(nightly_[^"]+)"')
+
+    version_str = tag_regex.match(tag_name).group(1)
 
     files = []
-    try:
-        with FTP(config["ftp"]["host"], config["ftp"]["user"], config["ftp"]["pass"]) as ftp:
-            version_str = tag_regex.match(tag_name).group(1)
-
-            path_template = config["ftp"]["path"]
-            path = path_template.format(type=build_type, version=version_str)
-            file_entries = list(ftp.mlsd(path, ["type"]))
-
-            for entry in file_entries:
-                if entry[1]["type"] == "file":
-                    files.append(entry[0])
-    except error_perm:
-        print("Received permanent FTP error!")
+    for mirror in config["ftp"]["mirrors"]:
+        url = mirror.format(type="nightly", version=version_str, file="")
+
+        try:
+            response = requests.get(url)
+            response.raise_for_status()
+        except Exception as ex:
+            print("Failed to retrieve filelist from %s: %s" % (url, ex))
+            continue
+
+        files = link_regex.findall(response.text)
+        break
+
+    if not files:
+        print("No files found!")
         return []
 
     out_data = []
@@ -91,13 +95,13 @@ def get_ftp_files(build_type, tag_name, config):
         # x64 is the name Visual Studio uses but Win64 works better for us since that gets displayed in the nightly post
         if "x64" in group_match:
             group_match = group_match.replace("x64", "Win64")
-        
+
         # nebula.py expects "MacOSX" as the group, but the build actions may pass off as just "Mac"
         if "Mac" == group_match:
             group_match = group_match.replace("Mac", "MacOSX")
 
         for mirror in config["ftp"]["mirrors"]:
-            download_url = mirror.format(type=build_type, version=version_str, file=file)
+            download_url = mirror.format(type="nightly", version=version_str, file=file)
             if primary_url is None:
                 primary_url = download_url
             else:

diff --git a/nightly.py b/nightly.py
@@ -65,7 +65,7 @@ def __init__(self):
 
     def post_build_actions(self):
         # Get the file list
-        files = file_list.get_ftp_files("nightly", self.tag_name, config)
+        files = file_list.get_nightly_files(self.tag_name, config)
 
         print("Generating installer manifests")
         for file in files:

diff --git a/test/test_ftp.py b/test/test_ftp.py
diff --git a/test/test_nightly_list.py b/test/test_nightly_list.py
@@ -0,0 +1,47 @@
+import unittest
+import requests_mock
+
+FILE_LISTING = '''<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<html>
+ <head>
+  <title>Index of /builds/nightly/test_tag</title>
+ </head>
+ <body>
+<h1>Index of /builds/nightly/test_tag</h1>
+  <table>
+   <tr><th valign="top"><img src="/icons/blank.gif" alt="[ICO]"></th><th><a href="?C=N;O=D">Name</a></th><th><a href="?C=M;O=A">Last modified</a></th><th><a href="?C=S;O=A">Size</a></th><th><a href="?C=D;O=A">Description</a></th></tr>
+   <tr><th colspan="5"><hr></th></tr>
+<tr><td valign="top"><img src="/icons/back.gif" alt="[PARENTDIR]"></td><td><a href="/builds/nightly/">Parent Directory</a></td><td>&nbsp;</td><td align="right">  - </td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/compressed.gif" alt="[   ]"></td><td><a href="nightly_test_tag-builds-Linux.tar.gz">nightly_test_tag-builds-Linux.tar.gz</a></td><td align="right">2023-04-13 06:25  </td><td align="right"> 30M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/compressed.gif" alt="[   ]"></td><td><a href="nightly_test_tag-builds-MacOSX.tar.gz">nightly_test_tag-builds-MacOSX.tar.gz</a></td><td align="right">2023-04-13 06:24  </td><td align="right"> 17M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/compressed.gif" alt="[   ]"></td><td><a href="nightly_test_tag-builds-Win32.zip">nightly_test_tag-builds-Win32.zip</a></td><td align="right">2023-04-13 06:43  </td><td align="right"> 29M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/compressed.gif" alt="[   ]"></td><td><a href="nightly_test_tag-builds-x64.zip">nightly_test_tag-builds-x64.zip</a></td><td align="right">2023-04-13 06:44  </td><td align="right"> 35M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="nightly_test_tag-debug-Win32.7z">nightly_test_tag-debug-Win32.7z</a></td><td align="right">2023-04-13 06:43  </td><td align="right"> 68M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="nightly_test_tag-debug-x64.7z">nightly_test_tag-debug-x64.7z</a></td><td align="right">2023-04-13 06:44  </td><td align="right"> 71M</td><td>&nbsp;</td></tr>
+   <tr><th colspan="5"><hr></th></tr>
+</table>
+<address>Apache/2.4.25 (Debian) Server at datacorder.porphyrion.feralhosting.com Port 80</address>
+</body></html>
+'''
+
+class NightlyListTestCase(unittest.TestCase):
+    @requests_mock.mock()
+    def test_file_listing(self, m):
+        m.get("http://01.example.org/builds/nightly/test_tag/", text=FILE_LISTING)
+
+        test_config = {
+            "ftp": {
+                "mirrors": [
+                    "http://01.example.org/builds/{type}/{version}/{file}",
+                    "http://02.example.org/builds/{type}/{version}/{file}",
+                ]
+            }
+        }
+
+        import file_list
+        files = file_list.get_nightly_files("nightly_test_tag", test_config)
+        self.assertEqual(4, len(files))
+
+
+if __name__ == '__main__':
+    unittest.main()