Skip to content

Commit

Permalink
feat: setup commands
Browse files Browse the repository at this point in the history
  • Loading branch information
lpmatos committed Jul 27, 2020
1 parent 6950455 commit 432013c
Show file tree
Hide file tree
Showing 8 changed files with 269 additions and 145 deletions.
13 changes: 13 additions & 0 deletions AUTHORS.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
=======
Credits
=======

Development Lead
----------------

* Lucca Pessoa da Silva Matos <luccapsm@gmail.com>

Contributors
------------

None yet. Why not be the first?
7 changes: 7 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ ARG PYTHON_VERSION=3.8-alpine3.11
FROM python:${PYTHON_VERSION} as base

FROM base as install-env
ARG GIT_PASSWORD

COPY [ "requirements.txt", "."]

Expand All @@ -12,6 +13,8 @@ RUN pip install --upgrade pip && \

FROM base

ENV GIT_PASSWORD=$GIT_PASSWORD

RUN set -ex && apk update

RUN apk add --update --no-cache \
Expand All @@ -24,4 +27,8 @@ WORKDIR /usr/src/code

COPY [ "./gitlab-clone", "." ]

RUN git config --global user.name "Lucca Pessoa" && \
git config --global user.email "luccapsm@gmail.com" && \
git config --global credential.helper cache

RUN find ./ -iname "*.py" -type f -exec chmod a+x {} \; -exec echo {} \;;
11 changes: 11 additions & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
include AUTHORS.rst
include CONTRIBUTING.rst
include HISTORY.rst
include LICENSE
include README.rst

recursive-include tests *
recursive-exclude * __pycache__
recursive-exclude * *.py[co]

recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif
61 changes: 61 additions & 0 deletions README.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
=================
gitlab-clone
=================


Tool for easy cloning whole gitlab structure to your local machine.


* Free software: MIT license



Requirements
------------

* Requests
* Python >= 3.6


Installation
------------

You can install "gitlab-clone" via `pip`_::

$ pip install gitlab-clone


Usage
-----


>>> gitlab-clone:
optional arguments:
-h, --help show this help message and exit
--group_id group_id Id of a group in gitlab
--token token Gitlab Token
--branch branch Branch to clone in all repos [by default master]
--gitlab-url gitlab Gitlab address [by default gitlab.com]


Example
-------

$ gitlab-clone --group=123 --token=MySecretToken --gitlab-url=gitlab.organization.com

For example if you clone this group https://gitlab.com/lmaolmaolmao which id is 6849299

you will have absolutely the same structure locally:

.. image:: https://github.com/ArseniyAntonov/gitlab-group-clone/raw/master/img/tree.png

>>> tree
.
└── lmaolmaolmao
└── subgroup1
└── subgroup2
└── subgroup3



.. _`pip`: https://pypi.python.org/pypi/pip/
2 changes: 1 addition & 1 deletion gitlab-clone/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
#!/usr/bin/env python
__version__ = "1.0.0"
264 changes: 134 additions & 130 deletions gitlab-clone/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,137 +16,141 @@ def pname():
pid = os.getpid()
return f"[gitlab-cloner - {str(pid)}]"

parser = optparse.OptionParser("usage: %prog [options]")

parser.add_option(
"-u",
"--url",
dest="url",
default="https://gitlab.com",
type="string",
help="base URL of the GitLab instance",
)

parser.add_option(
"-t", "--token", dest="token", default="", type="string", help="API token"
)

parser.add_option(
"-n",
"--namespace",
dest="namespace",
default="",
type="string",
help="namespace to clone",
)

parser.add_option(
"-p",
"--path",
dest="path",
default=os.getenv("PWD"),
type="string",
help="destination path for cloned projects",
)

parser.add_option(
"--disable-root",
action="store_true",
dest="noroot",
default=False,
help="do not create root namepace folder in path",
)

parser.add_option(
"--dry-run",
action="store_true",
dest="dryrun",
default=False,
help="list the repositories without clone/fetch",
)

(options, args) = parser.parse_args()

# TODO catch errrors
if not os.path.isdir(options.path):
sys.stderr.write("Error: destination path does not exist " + options.path + "\n")
exit(1)

git_path = shutil.which("git")
if git_path == "None":
sys.stderr.write("Error: git executable not installed or not in $PATH" + "\n")
exit(2)
else:
print(pname() + " using " + git_path)

t = time.time()

gl = gitlab.Gitlab(options.url, options.token)

group = gl.groups.get(options.namespace, lazy=True, include_subgroups=True)

projects = []

# Get all projects inside the namespace
for project in group.projects.list(all=True):
projects.append(project)
print(pname() + " found " + project.path_with_namespace)

# Get all projects inside the subgroups
for group in gl.groups.list(
all=True, owned=True, query_parameters={"id": options.namespace}
):

def main():
parser = optparse.OptionParser("usage: %prog [options]")

parser.add_option(
"-u",
"--url",
dest="url",
default="https://gitlab.com",
type="string",
help="base URL of the GitLab instance",
)

parser.add_option(
"-t", "--token", dest="token", default="", type="string", help="API token"
)

parser.add_option(
"-n",
"--namespace",
dest="namespace",
default="",
type="string",
help="namespace to clone",
)

parser.add_option(
"-p",
"--path",
dest="path",
default=os.getenv("PWD"),
type="string",
help="destination path for cloned projects",
)

parser.add_option(
"--disable-root",
action="store_true",
dest="noroot",
default=False,
help="do not create root namepace folder in path",
)

parser.add_option(
"--dry-run",
action="store_true",
dest="dryrun",
default=False,
help="list the repositories without clone/fetch",
)

(options, args) = parser.parse_args()

# TODO catch errrors
if not os.path.isdir(options.path):
sys.stderr.write("Error: destination path does not exist " + options.path + "\n")
exit(1)

git_path = shutil.which("git")
if git_path == "None":
sys.stderr.write("Error: git executable not installed or not in $PATH" + "\n")
exit(2)
else:
print(pname() + " using " + git_path)

t = time.time()

gl = gitlab.Gitlab(options.url, options.token)

group = gl.groups.get(options.namespace, lazy=True, include_subgroups=True)

projects = []

# Get all projects inside the namespace
for project in group.projects.list(all=True):
projects.append(project)
print(pname() + " found " + project.path_with_namespace)

subgroups = group.subgroups.list(all=True)
while True:
for subgroup in subgroups:

real_group = gl.groups.get(subgroup.id, lazy=True)

for project in real_group.projects.list(all=True):
projects.append(project)
print(pname() + " found " + project.path_with_namespace)

subgroups = real_group.subgroups.list(all=True)

if len(subgroups) == 0: next

if len(subgroups) == 0: break

if not options.dryrun:
for project in projects:
print(pname() + " clone/fetch project " + project.path_with_namespace)
folders = [f.strip().lower() for f in project.path_with_namespace.split("/")]
if options.noroot:
folders.remove(options.namespace)

mkdir = options.path
for i in range(len(folders) - 1):
mkdir = mkdir + "/" + folders[i]
if not os.path.isdir(mkdir):
os.mkdir(mkdir)

clone_path = options.path + "/" + "/".join(str(x) for x in folders)
clone_path = re.sub("/+", "/", clone_path)
print(pname() + " folder " + clone_path)
if not os.path.isdir(clone_path):
print(pname() + " cloning " + project.http_url_to_repo)
try:
subprocess.run(["git", "clone", project.http_url_to_repo, clone_path])
except:
sys.stderr.write("Unexpected error while cloning: terminating\n")
exit(2)
else:
print(pname() + " fetching " + project.http_url_to_repo)
try:
subprocess.run(["git", "-C", clone_path, "fetch", "--all"])
except:
sys.stderr.write("Unexpected error while fetching: terminating\n")
exit(3)

print(pname() + " mission accomplished in " + str(round(time.time() - t, 2)) + "s")
exit(0)
# Get all projects inside the subgroups
for group in gl.groups.list(
all=True, owned=True, query_parameters={"id": options.namespace}
):

for project in group.projects.list(all=True):
projects.append(project)
print(pname() + " found " + project.path_with_namespace)

subgroups = group.subgroups.list(all=True)
while True:
for subgroup in subgroups:

real_group = gl.groups.get(subgroup.id, lazy=True)

for project in real_group.projects.list(all=True):
projects.append(project)
print(pname() + " found " + project.path_with_namespace)

subgroups = real_group.subgroups.list(all=True)

if len(subgroups) == 0: next

if len(subgroups) == 0: break

if not options.dryrun:
for project in projects:
print(pname() + " clone/fetch project " + project.path_with_namespace)
folders = [f.strip().lower() for f in project.path_with_namespace.split("/")]
if options.noroot:
folders.remove(options.namespace)

mkdir = options.path
for i in range(len(folders) - 1):
mkdir = mkdir + "/" + folders[i]
if not os.path.isdir(mkdir):
os.mkdir(mkdir)

clone_path = options.path + "/" + "/".join(str(x) for x in folders)
clone_path = re.sub("/+", "/", clone_path)
print(pname() + " folder " + clone_path)
if not os.path.isdir(clone_path):
print(pname() + " cloning " + project.http_url_to_repo)
try:
subprocess.run(["git", "clone", project.http_url_to_repo, clone_path])
except:
sys.stderr.write("Unexpected error while cloning: terminating\n")
exit(2)
else:
print(pname() + " fetching " + project.http_url_to_repo)
try:
subprocess.run(["git", "-C", clone_path, "fetch", "--all"])
except:
sys.stderr.write("Unexpected error while fetching: terminating\n")
exit(3)

print(pname() + " mission accomplished in " + str(round(time.time() - t, 2)) + "s")
exit(0)

if __name__ == '__main__':
main()
Loading

0 comments on commit 432013c

Please sign in to comment.