diff --git a/backup_github/backup.py b/backup_github/backup.py index 4b12d41..8f73409 100644 --- a/backup_github/backup.py +++ b/backup_github/backup.py @@ -1,6 +1,7 @@ import logging import os import subprocess +from pathlib import Path from typing import Optional from backup_github.github import GithubAPI @@ -63,9 +64,6 @@ def backup_repositories(self): logging.debug(f"Repositories dir is {repo_dir}") logging.debug(f"Repositories: {self.repositories}") self.__save_repositories(self.repositories, repo_dir) - git_size.inc( - sum(os.path.getsize(f) for f in os.listdir(repo_dir) if os.path.isfile(f)) - ) def __get_repositories(self): return [repo["name"] for repo in self.api.get_repositories()] @@ -81,6 +79,12 @@ def __save_repositories(self, repositories, dir): ["id", "name", "private", "fork", "default_branch", "visibility"], f"{dir}/{repository}/repo.json", ) + git_size.labels(self.organization).inc( + sum( + p.stat().st_size + for p in Path(f"{dir}/{repository}/content").rglob("*") + ) + ) def __save_repo_content(self, repository, dir): cur_dir = os.getcwd() diff --git a/backup_github/main.py b/backup_github/main.py index ea6796f..a674cc2 100644 --- a/backup_github/main.py +++ b/backup_github/main.py @@ -1,22 +1,31 @@ import argparse import logging -import os import sys +from pathlib import Path from time import time from prometheus_client import write_to_textfile from backup_github.backup import Backup -from backup_github.metrics import backup_time, git_size, meta_size, registry, success +from backup_github.metrics import ( + backup_interval, + backup_time, + git_size, + meta_size, + registry, + success, +) from backup_github.parse_args import parse_args logging.basicConfig(level=logging.INFO) def main(): + start = time() parsed_args = None try: parsed_args = parse_args(sys.argv[1:]) + backup = Backup( parsed_args.token, parsed_args.organization, @@ -38,23 +47,20 @@ def main(): logging.info("Start backup of pulls") backup.backup_pulls() logging.info("Finish backup of pulls") - success.set(1) + success.labels(parsed_args.organization).set(1) except argparse.ArgumentError as e: logging.error(e.message) - success.set(0) + success.labels(parsed_args.organization).set(0) except AttributeError as e: logging.error(e) - success.set(0) + success.labels(parsed_args.organization).set(0) finally: - backup_time.set(int(time())) - meta_size.set( - sum( - os.path.getsize(f) - for f in os.listdir(parsed_args.output_dir) - if os.path.isfile(f) - ) - - git_size._value.get() + backup_time.labels(parsed_args.organization).set(int(time())) + meta_size.labels(parsed_args.organization).set( + sum(p.stat().st_size for p in Path(parsed_args.output_dir).rglob("*")) + - git_size.labels(parsed_args.organization)._value.get() ) + backup_interval.labels(parsed_args.organization).set(time() - start) write_to_textfile(f"{parsed_args.metrics_path}/github_backup.prom", registry) diff --git a/backup_github/metrics.py b/backup_github/metrics.py index 21d5cc6..f17f791 100644 --- a/backup_github/metrics.py +++ b/backup_github/metrics.py @@ -1,15 +1,33 @@ from prometheus_client import CollectorRegistry, Gauge registry = CollectorRegistry() -success = Gauge("github_backup_success", "1 if backup is okay", registry=registry) +success = Gauge( + "github_backup_success", + "1 if backup is okay", + labelnames=["organization"], + registry=registry, +) backup_time = Gauge( "github_backup_last_timestamp_seconds", "time of last backup in unixtime", + labelnames=["organization"], registry=registry, ) git_size = Gauge( - "github_backup_git_size_bytes", "Total size of git data", registry=registry + "github_backup_git_size_bytes", + "Total size of git data", + labelnames=["organization"], + registry=registry, ) meta_size = Gauge( - "github_backup_meta_size_bytes", "Total size of meta data", registry=registry + "github_backup_meta_size_bytes", + "Total size of meta data", + labelnames=["organization"], + registry=registry, +) +backup_interval = Gauge( + "github_backup_interval_timestamp_seconds", + "time of last backup in unixtime", + labelnames=["organization"], + registry=registry, )