Merge pull request #13 from emersonfelipesp/netbox-plugin

Closes #11 - Transform repository into a Netbox Plugin

.gitignore

@@ -1,5 +1,6 @@
 *.pyc
 *.swp
+*.egg-info
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*

netbox_plugins_store/__init__.py

@@ -0,0 +1,19 @@
+# Netbox plugin related import
+from extras.plugins import PluginConfig
+
+class PluginsStoreConfig(PluginConfig):
+    name = "netbox_plugins_store"
+    verbose_name = "Netbox Plugins Store"
+    description = "Easily find a useful Netbox Plugin for your environment!"
+    version = "0.0.1"
+    author = "Emerson Felipe (@emersonfelipesp) / Marcos Vella (@marcosvella)"
+    author_email = "emerson.felipe@nmultifibra.com.br"
+    base_url = "netbox_plugins_store"
+    required_settings = []
+    default_settings = {
+        'github_url': 'https://github.com',
+        'github_api_url': 'http://api.github.com',
+        'authorization': 'token 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',
+    }
+
+config = PluginsStoreConfig

netbox_plugins_store/github_api/__init__.py

@@ -0,0 +1 @@
+from . import github

netbox_plugins_store/github_api/github.py

@@ -1,57 +1,170 @@
 import requests
 import re
 import json
-import time
-
-github_url = 'https://github.com'
-github_api_url = 'http://api.github.com'
-
-page = requests.get(f'{github_url}/topics/netbox-plugin')
-text = page.text
-
-regex_match = re.findall(
-    r'Explore, go to repository, location:explore feed" href="\/[\w,\d\-]*\/[\w,\d\-]*"', text)
-
-# print(regex_match)
-repos = []
-for item in regex_match:
-    item = re.findall(r'\/[\w,\d\-]*\/[\w,\d\-]*', item)
-    repos.append(item[0])
-
-
-repositories = []
-f = open('repositories.json', 'w')
-for repo in repos:
-    data = requests.get(f'{github_api_url}/repos{repo}', headers={
-        "Authorization": f"token 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"
-    })
-    data = data.json()
-
-    info = {
-        "id": data["id"],
-        "name": data["name"],
-        "full_name": repo,
-        "owner": {
-            "login": data["owner"]["login"],
-            "id": data["owner"]["id"],
-            "url": data["owner"]["html_url"],
-            "type": data["owner"]["type"]
-        },
-        "url": data["html_url"],
-        "description": data["description"],
-        "git_url": data["git_url"],
-        "clone_url": data["clone_url"],
-        "created_at": data["created_at"],
-        "updated_at": data["updated_at"],
-        "pushed_at": data["pushed_at"],
-        "stargazers_count": data["stargazers_count"],
-        "watchers_count": data["watchers_count"],
-        "language": data["language"],
-        "forks": data["forks"],
-        "open_issues": data["open_issues"],
-        "topics": data["topics"]
-    }
-
-    repositories.append(info)
-
-f.write(json.dumps(repositories))
+import math
+
+
+class GitHubAPI():
+    def get_netbox_repos(self):
+
+        github_user = 'emersonfelipesp'
+        github_repo_name = 'netbox-plugins-store'
+        github_file_name = 'repositories.json'
+
+        # HTTP Request
+        get_response = requests.get(
+            f'https://api.github.com/repos/{github_user}/{github_repo_name}/contents/{github_file_name}',
+            headers = {
+                "Accept": "application/vnd.github.v3+json"
+            }
+        ).json()
+
+        raw_content_url = get_response.get('download_url')
+
+        # Get Raw Content of 'repositories.json' file from GitHub 'emersonfelipesp/netbox-plugins-store' repo
+        get_json_file = requests.get(
+            raw_content_url
+        )
+
+        # Convert HTTP response to 'dict'
+        repositories = json.loads(get_json_file.text)
+        return repositories
+
+
+
+
+    def create_netbox_repos(self):
+        github_url = self.url
+        github_api_url = self.api_url
+
+        # Get Github topics/netbox-plugin page
+        page = requests.get(f'{github_url}/topics/netbox-plugin')
+        text = page.text
+
+        # Look through page content for the number of public repositories
+        repos_number = re.findall(r'\d{2} public repositories', text)
+        repos_number = re.findall(r'\d{2}', repos_number[0])[0]
+
+        # Define number of pages that should be fetched
+        number_of_pages = int(math.ceil(int(repos_number) / 30))
+        if number_of_pages == 0:
+            number_of_pages = 1
+
+        # Create empty array to received repositories found on page
+        repos = []
+
+        # Loop until fetch all pages needed to match the number of repos (30 repos per page)
+        i = 1
+        while number_of_pages >= i:
+
+            # If not first page (results are already acccessable from the first request), fetch i'th page
+            if not i == 1:
+                page = requests.get(
+                    f'https://github.com/topics/netbox-plugin?page={i}')
+                text = page.text
+
+            # Search for this regex match on page results
+            regex_match = re.findall(
+                r'Explore, go to repository, location:explore feed" href="\/[\w,\d\-]*\/[\w,\d\-]*"', text)
+
+            # Loop through every result, separating regular text from repo /author/name
+            for item in regex_match:
+                item = re.findall(r'\/[\w,\d\-]*\/[\w,\d\-]*', item)
+                repos.append(item[0])
+
+            i += 1
+
+        repositories = []
+        for repo in repos:
+            # HTTP Request
+            data = requests.get(
+                f'http://api.github.com/repos{repo}',
+                headers={
+                    "Authorization": f"token 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"
+                }
+            )
+
+            # Convert HTTP Response to JSON
+            data = data.json()
+
+            if data.get('message') == 'Bad credentials':
+                print("Token is wrong.")
+                return
+
+            info = {
+                "id": data["id"],
+                "name": data["name"],
+                "full_name": repo,
+                "owner": {
+                    "login": data["owner"]["login"],
+                    "id": data["owner"]["id"],
+                    "url": data["owner"]["html_url"],
+                    "type": data["owner"]["type"]
+                },
+                "url": data["html_url"],
+                "description": data["description"],
+                "git_url": data["git_url"],
+                "clone_url": data["clone_url"],
+                "created_at": data["created_at"],
+                "updated_at": data["updated_at"],
+                "pushed_at": data["pushed_at"],
+                "stargazers_count": data["stargazers_count"],
+                "watchers_count": data["watchers_count"],
+                "language": data["language"],
+                "forks": data["forks"],
+                "open_issues": data["open_issues"],
+                "topics": data["topics"]
+            }
+
+            repositories.append(info)
+
+        return repositories
+
+    def __init__(self, url, api_url):
+
+        # GitHub Parameters to run API
+        self.url = url
+        self.api_url = api_url
+
+
+def write_json_file(repos_json):
+    # Open file
+    f = open('repositories_fixed.json', 'w')
+
+    # Write returned JSON to file
+    f.write(repos_json)
+
+
+def check_for_json_file():
+    '''If JSON file exists, return it. Else creates it and save to file.'''
+
+    try:
+        f = open('repositories_fixed.json')
+        print('File found.')
+
+        # Read file
+        repositories = f.read()
+
+        # Convert 'str' to 'dict'
+        repositories = json.loads(repositories)
+
+    except FileNotFoundError:
+        print('File not Found. Creating it.')
+
+        # Calls GitHubAPI's create_netbox_repos() method
+        repositories = GitHubAPI(
+            'https://github.com',
+            'http://api.github.com',
+        ).create_netbox_repos()
+
+        # Save retured JSON from GitHub to 'repositories_fixed.json' file
+        write_file = write_json_file(json.dumps(repositories))
+
+    return repositories
+
+
+# JSON returned from GitHub
+repositories = GitHubAPI(
+    'https://github.com',
+    'http://api.github.com',
+).get_netbox_repos()