Add ability to add github download links to autogenerated netkans fro…

…m spacedock

netkan/netkan/spacedock_adder.py

@@ -14,6 +14,10 @@
 from .mod_analyzer import ModAnalyzer
 from .queue_handler import BaseMessageHandler, QueueHandler
 from .repos import NetkanRepo
+from github.Repository import Repository
+from github.GitRelease import GitRelease
+from github.GitReleaseAsset import GitReleaseAsset
+from github import Github
 
 if TYPE_CHECKING:
     from mypy_boto3_sqs.service_resource import Message
@@ -29,6 +33,7 @@ class SpaceDockAdder:
     PR_BODY_TEMPLATE = Template(read_text('netkan', 'sd_adder_pr_body_template.md'))
     USER_TEMPLATE = Template('[$username]($user_url)')
     TITLE_TEMPLATE = Template('Add $name from $site_name')
+    NETKAN_SEPARATOR = '---\n'
     _info: Dict[str, Any]
 
     def __init__(self, message: Message, nk_repo: NetkanRepo, game: Game, github_pr: Optional[GitHubPR] = None) -> None:
@@ -52,16 +57,16 @@ def try_add(self) -> bool:
         netkan = self.make_netkan(self.info)
 
         # Create .netkan file or quit if already there
-        netkan_path = self.nk_repo.nk_path(netkan.get('identifier', ''))
+        netkan_path = self.nk_repo.nk_path(netkan[0].get('identifier', ''))
         if netkan_path.exists():
             # Already exists, we are done
             return True
 
         # Create and checkout branch
-        branch_name = f"add/{netkan.get('identifier')}"
+        branch_name = f"add/{netkan[0].get('identifier')}"
         with self.nk_repo.change_branch(branch_name):
             # Create file
-            netkan_path.write_text(self.yaml_dump(netkan))
+            netkan_path.write_text(SpaceDockAdder.NETKAN_SEPARATOR.join([self.yaml_dump(single_netkan) for single_netkan in netkan]))
 
             # Add netkan to branch
             self.nk_repo.git_repo.index.add([netkan_path.as_posix()])
@@ -111,8 +116,18 @@ def yaml_dump(self, obj: Dict[str, Any]) -> str:
     def sd_download_url(info: Dict[str, Any]) -> str:
         return f"https://spacedock.info/mod/{info.get('id', '')}/{info.get('name', '')}/download"
 
-    def make_netkan(self, info: Dict[str, Any]) -> Dict[str, Any]:
+    def make_netkan(self, info: Dict[str, Any]) -> List[Dict[str, Any]]:
+        netkans = []
         ident = re.sub(r'[\W_]+', '', info.get('name', ''))
+        gh_repo = self.get_github_repo(info.get('source_link',''))
+        if gh_repo != None:
+            gh_netkan = self.make_github_netkan(ident,gh_repo,info)
+            if gh_netkan != None:
+                netkans.append(gh_netkan)
+        netkans.append(self.make_spacedock_netkan(ident,info))
+        return netkans
+
+    def make_spacedock_netkan(self, ident: str, info: Dict[str, Any]) -> Dict[str, Any]:
         mod: Optional[ModAnalyzer] = None
         props: Dict[str, Any] = {}
         url = SpaceDockAdder.sd_download_url(info)
@@ -125,7 +140,7 @@ def make_netkan(self, info: Dict[str, Any]) -> Dict[str, Any]:
                           self.__class__.__name__, ident, url, exc_info=exc)
         vref_props = {'$vref': props.pop('$vref')} if '$vref' in props else {}
         return {
-            'spec_version': 'v1.18',
+            'spec_version': 'v1.34',
             'identifier': ident,
             '$kref': f"#/ckan/spacedock/{info.get('id', '')}",
             **(vref_props),
@@ -134,6 +149,62 @@ def make_netkan(self, info: Dict[str, Any]) -> Dict[str, Any]:
             'x_via': f"Automated {info.get('site_name')} CKAN submission"
         }
 
+
+    def get_github_repo(self, source_link: str) -> Optional[Repository]:
+        if 'github.com/' in source_link:
+            sections = source_link.strip('/').split('/')
+            repo_name = sections[-2] + '/' + sections[-1]
+            g = Github()
+            try:
+                return g.get_repo(repo_name)
+            except Exception as exc:
+                logging.error('%s failed to get the github repository from spacedock source url %s', self.__class__.__name__, source_link, exc_info=exc)
+                return None
+        else:
+            return None
+
+    def make_github_netkan(self, ident: str, gh_repo: Repository, info: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        mod: Optional[ModAnalyzer] = None
+        props: Dict[str, Any] = {}
+        releases = gh_repo.get_releases()
+        if releases.totalCount == 0:
+            return None
+        latest_release: GitRelease = releases[0]
+        tag_name = latest_release.tag_name
+        digit = re.search(r"\d", tag_name)
+        version_find = ''
+        if digit:
+            version_find = tag_name[:digit.start()]
+        assets = latest_release.assets
+        if len(assets) == 0:
+            return None
+        url = assets[0].browser_download_url
+        try:
+            mod = ModAnalyzer(ident, url, self.game)
+            props = mod.get_netkan_properties() if mod else {}
+        except Exception as exc:  # pylint: disable=broad-except
+            # Tell Discord about the problem and move on
+            logging.error('%s failed to analyze %s from %s',
+                          self.__class__.__name__, ident, url, exc_info=exc)
+        vref_props = {'$vref': props.pop('$vref')} if '$vref' in props else {}
+        netkan = {
+            'spec_version': 'v1.34',
+            'identifier': ident,
+            '$kref': f"#/ckan/github/{gh_repo.full_name}",
+            **(vref_props),
+            'license': info.get('license', '').strip().replace(' ', '-'),
+            **(props),
+        }
+
+        if version_find != '':
+            netkan['x_netkan_version_edit'] = {
+                'find': f'^{version_find}',
+                'replace': '',
+                'strict': 'false'
+            }
+        return netkan
+
+
     @property
     def delete_attrs(self) -> DeleteMessageBatchRequestEntryTypeDef:
         return {