#45, #47: Empty repositories and metrics path (#46)

backup_github/backup.py

@@ -1,7 +1,6 @@
 import logging
 import os
 import subprocess
-from pathlib import Path
 from typing import Optional
 
 from backup_github.github import GithubAPI
@@ -64,15 +63,19 @@ def backup_repositories(self):
         logging.debug(f"Repositories dir is {repo_dir}")
         logging.debug(f"Repositories: {self.repositories}")
         self.__save_repositories(self.repositories, repo_dir)
-        git_size.inc(sum(p.stat().st_size for p in Path(repo_dir).rglob("*")))
+        git_size.inc(
+            sum(os.path.getsize(f) for f in os.listdir(repo_dir) if os.path.isfile(f))
+        )
 
     def __get_repositories(self):
         return [repo["name"] for repo in self.api.get_repositories()]
 
     def __save_repositories(self, repositories, dir):
         for repository in repositories:
+            if self.api.get_repository(repository)["size"] == 0:
+                continue
             self.__save_repo_content(repository, dir)
-            repo = self.api.get_repo(repository)
+            repo = self.api.get_repository(repository)
             filter_save(
                 repo,
                 ["id", "name", "private", "fork", "default_branch", "visibility"],

backup_github/github.py

@@ -31,20 +31,18 @@ def __init__(self, message=None):
     def raise_by_status(self, response):
         if response.status_code == 403:
             logging.warning("Status is 403 - Rate limit exceeded exception")
-            raise self.RateLimitExceededException(
-                json.loads(response.content)["message"]
-            )
+            raise self.RateLimitExceededException(json.loads(response.content))
         elif response.status_code == 404:
             logging.warning(
                 f"Status is {response.status_code} - Client error: Not found"
             )
-            raise self.ClientError(json.loads(response.content)["message"])
+            raise self.ClientError(json.loads(response.content))
         elif 400 <= response.status_code < 500:
             logging.warning(f"Status is {response.status_code} - Client error")
-            raise self.ClientError(json.loads(response.content)["message"])
+            raise self.ClientError(json.loads(response.content))
         elif 500 <= response.status_code < 600:
             logging.warning(f"Status is {response.status_code} - Server error")
-            raise self.ServerError(json.loads(response.content)["message"])
+            raise self.ServerError(json.loads(response.content))
 
     def retry(func):
         def ret(self, *args, **kwargs):
@@ -122,7 +120,7 @@ def get_member_status(self, member_login):
             f"https://api.github.com/orgs/{self.organization}/memberships/{member_login}"
         )
 
-    def get_repo(self, repo_name):
+    def get_repository(self, repo_name):
         return self.make_request(
             f"https://api.github.com/repos/{self.organization}/{repo_name}"
         )

backup_github/main.py

@@ -1,12 +1,13 @@
 import argparse
 import logging
+import os
 import sys
-from pathlib import Path
+from time import time
 
 from prometheus_client import write_to_textfile
 
 from backup_github.backup import Backup
-from backup_github.metrics import git_size, meta_size, registry, success, time
+from backup_github.metrics import backup_time, git_size, meta_size, registry, success
 from backup_github.parse_args import parse_args
 
 logging.basicConfig(level=logging.INFO)
@@ -45,12 +46,16 @@ def main():
         logging.error(e)
         success.set(0)
     finally:
-        time.set_to_current_time()
+        backup_time.set(int(time()))
         meta_size.set(
-            sum(p.stat().st_size for p in Path(parsed_args.output_dir).rglob("*"))
+            sum(
+                os.path.getsize(f)
+                for f in os.listdir(parsed_args.output_dir)
+                if os.path.isfile(f)
+            )
             - git_size._value.get()
         )
-        write_to_textfile("backup_github.prom", registry)
+        write_to_textfile(f"{parsed_args.metrics_path}/github_backup.prom", registry)
 
 
 if __name__ == "__main__":

backup_github/metrics.py

@@ -2,7 +2,7 @@
 
 registry = CollectorRegistry()
 success = Gauge("github_backup_success", "1 if backup is okay", registry=registry)
-time = Gauge(
+backup_time = Gauge(
     "github_backup_last_timestamp_seconds",
     "time of last backup in unixtime",
     registry=registry,

backup_github/parse_args.py

@@ -63,5 +63,11 @@ def parse_args(args=None) -> argparse.Namespace:
         dest="all",
         help="run backup of all data",
     )
+    parser.add_argument(
+        "--metrics_path",
+        default="/var/lib/node_exporter",
+        dest="metrics_path",
+        help="path for .prom file with metrics",
+    )
     parsed = parser.parse_args(args)
     return parsed