Merge branch 'main' into release

backup_github/backup.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import subprocess
+from pathlib import Path
 from typing import Optional
 
 from backup_github.github import GithubAPI
@@ -63,9 +64,6 @@ def backup_repositories(self):
         logging.debug(f"Repositories dir is {repo_dir}")
         logging.debug(f"Repositories: {self.repositories}")
         self.__save_repositories(self.repositories, repo_dir)
-        git_size.inc(
-            sum(os.path.getsize(f) for f in os.listdir(repo_dir) if os.path.isfile(f))
-        )
 
     def __get_repositories(self):
         return [repo["name"] for repo in self.api.get_repositories()]
@@ -81,6 +79,12 @@ def __save_repositories(self, repositories, dir):
                 ["id", "name", "private", "fork", "default_branch", "visibility"],
                 f"{dir}/{repository}/repo.json",
             )
+            git_size.labels(self.organization).inc(
+                sum(
+                    p.stat().st_size
+                    for p in Path(f"{dir}/{repository}/content").rglob("*")
+                )
+            )
 
     def __save_repo_content(self, repository, dir):
         cur_dir = os.getcwd()

backup_github/main.py

@@ -1,22 +1,31 @@
 import argparse
 import logging
-import os
 import sys
+from pathlib import Path
 from time import time
 
 from prometheus_client import write_to_textfile
 
 from backup_github.backup import Backup
-from backup_github.metrics import backup_time, git_size, meta_size, registry, success
+from backup_github.metrics import (
+    backup_interval,
+    backup_time,
+    git_size,
+    meta_size,
+    registry,
+    success,
+)
 from backup_github.parse_args import parse_args
 
 logging.basicConfig(level=logging.INFO)
 
 
 def main():
+    start = time()
     parsed_args = None
     try:
         parsed_args = parse_args(sys.argv[1:])
+
         backup = Backup(
             parsed_args.token,
             parsed_args.organization,
@@ -38,23 +47,20 @@ def main():
             logging.info("Start backup of pulls")
             backup.backup_pulls()
             logging.info("Finish backup of pulls")
-        success.set(1)
+        success.labels(parsed_args.organization).set(1)
     except argparse.ArgumentError as e:
         logging.error(e.message)
-        success.set(0)
+        success.labels(parsed_args.organization).set(0)
     except AttributeError as e:
         logging.error(e)
-        success.set(0)
+        success.labels(parsed_args.organization).set(0)
     finally:
-        backup_time.set(int(time()))
-        meta_size.set(
-            sum(
-                os.path.getsize(f)
-                for f in os.listdir(parsed_args.output_dir)
-                if os.path.isfile(f)
-            )
-            - git_size._value.get()
+        backup_time.labels(parsed_args.organization).set(int(time()))
+        meta_size.labels(parsed_args.organization).set(
+            sum(p.stat().st_size for p in Path(parsed_args.output_dir).rglob("*"))
+            - git_size.labels(parsed_args.organization)._value.get()
         )
+        backup_interval.labels(parsed_args.organization).set(time() - start)
         write_to_textfile(f"{parsed_args.metrics_path}/github_backup.prom", registry)
 
 

backup_github/metrics.py

@@ -1,15 +1,33 @@
 from prometheus_client import CollectorRegistry, Gauge
 
 registry = CollectorRegistry()
-success = Gauge("github_backup_success", "1 if backup is okay", registry=registry)
+success = Gauge(
+    "github_backup_success",
+    "1 if backup is okay",
+    labelnames=["organization"],
+    registry=registry,
+)
 backup_time = Gauge(
     "github_backup_last_timestamp_seconds",
     "time of last backup in unixtime",
+    labelnames=["organization"],
     registry=registry,
 )
 git_size = Gauge(
-    "github_backup_git_size_bytes", "Total size of git data", registry=registry
+    "github_backup_git_size_bytes",
+    "Total size of git data",
+    labelnames=["organization"],
+    registry=registry,
 )
 meta_size = Gauge(
-    "github_backup_meta_size_bytes", "Total size of meta data", registry=registry
+    "github_backup_meta_size_bytes",
+    "Total size of meta data",
+    labelnames=["organization"],
+    registry=registry,
+)
+backup_interval = Gauge(
+    "github_backup_interval_timestamp_seconds",
+    "time of last backup in unixtime",
+    labelnames=["organization"],
+    registry=registry,
 )