-
Notifications
You must be signed in to change notification settings - Fork 1
/
update_git_repos.py
95 lines (73 loc) · 2.64 KB
/
update_git_repos.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
#!/usr/bin/python
# -*- coding: utf-8 -*-
import functools
import multiprocessing
from multiprocessing import pool
import os
import subprocess
# BASE_GIT = 'git://git.openstack.org' # DEPRECATED
BASE_GIT = 'https://opendev.org'
# Where the repositories are stored.
BASE_DIR, _ = os.path.split(os.path.abspath(__file__))
BASE_DIR += '/'
REPO_FILE = BASE_DIR + 'git_repos.txt'
MAX_THREADS = 2
def _execute_commands(cmds, directory, action):
for cmd in cmds:
try:
# 10 mins max.
return subprocess.check_output(
cmd, stderr=subprocess.STDOUT, cwd=directory, timeout=600)
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as \
error:
print('Error during %s: %s' % (action, str(error.output)))
return False
def _git_update(repo_dir):
print('--> Updating %s' % repo_dir)
cmds = [['git', 'fetch', 'origin']]
return _execute_commands(cmds, repo_dir, 'update')
def _git_clone(repo_base_dir, repository):
print('--> Cloning %s' % repository)
cmds = [['git', 'clone', '--mirror', repository]]
return _execute_commands(cmds, repo_base_dir, 'clone')
def _remove_directory(repo_dir):
print('--> Removing %s' % repo_dir)
cmds = [['rm', '-fr', repo_dir]]
return _execute_commands(cmds, None, 'clone')
def update_or_clone(_lock, *repo_tuple):
repository, repo_base_dir, repo_dir = tuple(*repo_tuple)
res = None
if os.path.exists(repo_dir):
res = _git_update(repo_dir)
if res is False:
_remove_directory(repo_dir)
if not os.path.exists(repo_dir):
with _lock:
if not os.path.exists(repo_base_dir):
os.mkdir(repo_base_dir)
_git_clone(repo_base_dir, repository)
def gen_repos(repositories):
for repo in repositories:
repo_base_dir = BASE_DIR + repo[0] + '/'
repo_dir = repo_base_dir + repo[1] + '/'
repository = BASE_GIT + '/' + repo[0] + '/' + repo[1]
yield repository, repo_base_dir, repo_dir
print('Updating/cloning repositories.')
# Read repo list
repositories = []
with open(REPO_FILE, 'r') as f:
for line in f:
line = line.splitlines()[0]
if line.startswith('#'):
continue
repo = line.split('/')
if len(repo) != 2:
continue
repo_base = repo[0]
repo_name = repo[1]
repositories.append((repo_base, repo_name))
_lock = multiprocessing.Lock()
tpool = pool.ThreadPool(MAX_THREADS)
_update_or_clone = functools.partial(update_or_clone, _lock)
tpool.map(_update_or_clone, gen_repos(repositories))
print('Git repositories updated! Process finished.')