diff --git a/AUTHORS.rst b/AUTHORS.rst index f1bfe39..a2988a4 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -1,13 +1,13 @@ -======= -Credits -======= - -Development Lead ----------------- - -* Lucca Pessoa da Silva Matos - -Contributors ------------- - -None yet. Why not be the first? +======= +Credits +======= + +Development Lead +---------------- + +* Lucca Pessoa da Silva Matos + +Contributors +------------ + +None yet. Why not be the first? diff --git a/Dockerfile b/Dockerfile index 39ccef6..a573e60 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,34 +1,34 @@ -ARG PYTHON_VERSION=3.8-alpine3.11 - -FROM python:${PYTHON_VERSION} as base - -FROM base as install-env -ARG GIT_PASSWORD - -COPY [ "requirements.txt", "."] - -RUN pip install --upgrade pip && \ - pip install --upgrade setuptools && \ - pip install --user --no-warn-script-location -r ./requirements.txt - -FROM base - -ENV GIT_PASSWORD=$GIT_PASSWORD - -RUN set -ex && apk update - -RUN apk add --update --no-cache \ - bash=5.0.11-r1 \ - git=2.24.3-r0 - -COPY --from=install-env [ "/root/.local", "/usr/local" ] - -WORKDIR /usr/src/code - -COPY [ "./gitlab-clone", "." ] - -RUN git config --global user.name "Lucca Pessoa" && \ - git config --global user.email "luccapsm@gmail.com" && \ - git config --global credential.helper cache - -RUN find ./ -iname "*.py" -type f -exec chmod a+x {} \; -exec echo {} \;; +ARG PYTHON_VERSION=3.8-alpine3.11 + +FROM python:${PYTHON_VERSION} as base + +FROM base as install-env +ARG GIT_PASSWORD + +COPY [ "requirements.txt", "."] + +RUN pip install --upgrade pip && \ + pip install --upgrade setuptools && \ + pip install --user --no-warn-script-location -r ./requirements.txt + +FROM base + +ENV GIT_PASSWORD=$GIT_PASSWORD + +RUN set -ex && apk update + +RUN apk add --update --no-cache \ + bash=5.0.11-r1 \ + git=2.24.3-r0 + +COPY --from=install-env [ "/root/.local", "/usr/local" ] + +WORKDIR /usr/src/code + +COPY [ "./gitlab-clone", "." ] + +RUN git config --global user.name "Lucca Pessoa" && \ + git config --global user.email "luccapsm@gmail.com" && \ + git config --global credential.helper cache + +RUN find ./ -iname "*.py" -type f -exec chmod a+x {} \; -exec echo {} \;; diff --git a/MANIFEST.in b/MANIFEST.in index 965b2dd..83a34cc 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,11 +1,11 @@ -include AUTHORS.rst -include CONTRIBUTING.rst -include HISTORY.rst -include LICENSE -include README.rst - -recursive-include tests * -recursive-exclude * __pycache__ -recursive-exclude * *.py[co] - -recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif +include AUTHORS.rst +include CONTRIBUTING.rst +include HISTORY.rst +include LICENSE +include README.rst + +recursive-include tests * +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] + +recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif diff --git a/README.md b/README.md index d6e175e..9b722c4 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,10 @@ commitlint

-

Welcome to GitLab-Clone repository

+

Welcome to GitLab Clone Recursive repository

- Automate clonning projects and groups in GitLab + Automate clone your projects in GitLab

@@ -51,20 +51,26 @@ If you want use this repository you need to make a **git clone**: ```bash -git clone --depth 1 https://github.com/lpmatos/gitlab-clone.git -b master +git clone --depth 1 https://github.com/lpmatos/gitlab-clone-recursive.git -b master ``` This will give access on your **local machine**. ### Pre-Requisites -To this project you yeed: +To **develop** in this project you yeed: * NPM | Yarn (package tool) * Install Packages - * Husky - * Commitlint - * Commitizen + * NodeJS + * Python +* Docker and Docker Compose + +To **use** this project you yeed: + +* Packages +* Git Installation and Configuration +* Install Package gitlab-clone-recursive ### How to use it? diff --git a/README.rst b/README.rst deleted file mode 100644 index 268e33e..0000000 --- a/README.rst +++ /dev/null @@ -1,38 +0,0 @@ -================= -gitlab-clone-recursive -================= - - -Tool for easy cloning whole gitlab structure to your local machine. - - -* Free software: MIT license - - - -Requirements ------------- - -* Python-GitLab -* Python >= 3.6 - - -Installation ------------- - -You can install "gitlab-clone-recursive" via `pip`_:: - - $ pip install gitlab-clone-recursive - - -Usage ------ - - ->>> gitlab-clone-recursive: - optional arguments: - -h, --help show this help message and exit - --group_id group_id Id of a group in gitlab - --token token Gitlab Token - --branch branch Branch to clone in all repos [by default master] - --gitlab-url gitlab Gitlab address [by default gitlab.com] diff --git a/commitlint.config.js b/commitlint.config.js index a0832de..47d10ad 100644 --- a/commitlint.config.js +++ b/commitlint.config.js @@ -1 +1 @@ -module.exports = {extends: ['@commitlint/config-conventional']} +module.exports = {extends: ['@commitlint/config-conventional']} diff --git a/docker-compose.yml b/docker-compose.yml index 8453357..e2c1be1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,28 +1,28 @@ -version: "3.8" - -# ============================================================================== -# ANCHORS -# ============================================================================== - -x-logging: &default-logging - driver: "json-file" - options: - max-size: "500k" - max-file: "20" - -# ============================================================================== -# SERVICES -# ============================================================================== - -services: - - gitlab-clone: - container_name: ${CONTAINER_NAME} - env_file: [ "./.env" ] - build: - context: ./${PATH_DOCKERFILE} - dockerfile: Dockerfile - stdin_open: true - tty: true - restart: on-failure - logging: *default-logging +version: "3.8" + +# ============================================================================== +# ANCHORS +# ============================================================================== + +x-logging: &default-logging + driver: "json-file" + options: + max-size: "500k" + max-file: "20" + +# ============================================================================== +# SERVICES +# ============================================================================== + +services: + + gitlab-clone: + container_name: ${CONTAINER_NAME} + env_file: [ "./.env" ] + build: + context: ./${PATH_DOCKERFILE} + dockerfile: Dockerfile + stdin_open: true + tty: true + restart: on-failure + logging: *default-logging diff --git a/gitlab-clone-recursive/__init__.py b/gitlab/__init__.py similarity index 95% rename from gitlab-clone-recursive/__init__.py rename to gitlab/__init__.py index f102a9c..362d74a 100644 --- a/gitlab-clone-recursive/__init__.py +++ b/gitlab/__init__.py @@ -1 +1 @@ -__version__ = "0.0.1" +__version__ = "0.0.1" diff --git a/gitlab-clone-recursive/cli.py b/gitlab/cli.py similarity index 96% rename from gitlab-clone-recursive/cli.py rename to gitlab/cli.py index d2ce6b8..4465de8 100644 --- a/gitlab-clone-recursive/cli.py +++ b/gitlab/cli.py @@ -1,154 +1,154 @@ -import os -import sys -import re -import shutil -import time -import gitlab -import optparse -import subprocess - -def pname(): - pid = os.getpid() - return f"[gitlab-cloner - {str(pid)}]" - -def main(): - parser = optparse.OptionParser("usage: %prog [options]") - - parser.add_option( - "-u", - "--url", - dest="url", - default="https://gitlab.com", - type="string", - help="base URL of the GitLab instance", - ) - - parser.add_option( - "-t", "--token", dest="token", default="", type="string", help="API token" - ) - - parser.add_option( - "-n", - "--namespace", - dest="namespace", - default="", - type="string", - help="namespace to clone", - ) - - parser.add_option( - "-p", - "--path", - dest="path", - default=os.getenv("PWD"), - type="string", - help="destination path for cloned projects", - ) - - parser.add_option( - "--disable-root", - action="store_true", - dest="noroot", - default=False, - help="do not create root namepace folder in path", - ) - - parser.add_option( - "--dry-run", - action="store_true", - dest="dryrun", - default=False, - help="list the repositories without clone/fetch", - ) - - (options, args) = parser.parse_args() - - clone(options) - -def clone(options): - # TODO catch errrors - if not os.path.isdir(options.path): - sys.stderr.write("Error: destination path does not exist " + options.path + "\n") - exit(1) - - git_path = shutil.which("git") - if git_path == "None": - sys.stderr.write("Error: git executable not installed or not in $PATH" + "\n") - exit(2) - else: - print(pname() + " using " + git_path) - - t = time.time() - - gl = gitlab.Gitlab(options.url, options.token) - - group = gl.groups.get(options.namespace, lazy=True, include_subgroups=True) - - projects = [] - - # Get all projects inside the namespace - for project in group.projects.list(all=True): - projects.append(project) - print(pname() + " found " + project.path_with_namespace) - - # Get all projects inside the subgroups - for group in gl.groups.list( - all=True, owned=True, query_parameters={"id": options.namespace} - ): - - for project in group.projects.list(all=True): - projects.append(project) - print(pname() + " found " + project.path_with_namespace) - - subgroups = group.subgroups.list(all=True) - while True: - for subgroup in subgroups: - - real_group = gl.groups.get(subgroup.id, lazy=True) - - for project in real_group.projects.list(all=True): - projects.append(project) - print(pname() + " found " + project.path_with_namespace) - - subgroups = real_group.subgroups.list(all=True) - - if len(subgroups) == 0: next - - if len(subgroups) == 0: break - - if not options.dryrun: - for project in projects: - print(pname() + " clone/fetch project " + project.path_with_namespace) - folders = [f.strip().lower() for f in project.path_with_namespace.split("/")] - if options.noroot: - folders.remove(options.namespace) - - mkdir = options.path - for i in range(len(folders) - 1): - mkdir = mkdir + "/" + folders[i] - if not os.path.isdir(mkdir): - os.mkdir(mkdir) - - clone_path = options.path + "/" + "/".join(str(x) for x in folders) - clone_path = re.sub("/+", "/", clone_path) - print(pname() + " folder " + clone_path) - if not os.path.isdir(clone_path): - print(pname() + " cloning " + project.http_url_to_repo) - try: - subprocess.run(["git", "clone", project.http_url_to_repo, clone_path]) - except: - sys.stderr.write("Unexpected error while cloning: terminating\n") - exit(2) - else: - print(pname() + " fetching " + project.http_url_to_repo) - try: - subprocess.run(["git", "-C", clone_path, "fetch", "--all"]) - except: - sys.stderr.write("Unexpected error while fetching: terminating\n") - exit(3) - - print(pname() + " mission accomplished in " + str(round(time.time() - t, 2)) + "s") - exit(0) - -if __name__ == '__main__': - main() +import os +import sys +import re +import shutil +import time +import gitlab +import optparse +import subprocess + +def pname(): + pid = os.getpid() + return f"[gitlab-cloner - {str(pid)}]" + +def main(): + parser = optparse.OptionParser("usage: %prog [options]") + + parser.add_option( + "-u", + "--url", + dest="url", + default="https://gitlab.com", + type="string", + help="base URL of the GitLab instance", + ) + + parser.add_option( + "-t", "--token", dest="token", default="", type="string", help="API token" + ) + + parser.add_option( + "-n", + "--namespace", + dest="namespace", + default="", + type="string", + help="namespace to clone", + ) + + parser.add_option( + "-p", + "--path", + dest="path", + default=os.getenv("PWD"), + type="string", + help="destination path for cloned projects", + ) + + parser.add_option( + "--disable-root", + action="store_true", + dest="noroot", + default=False, + help="do not create root namepace folder in path", + ) + + parser.add_option( + "--dry-run", + action="store_true", + dest="dryrun", + default=False, + help="list the repositories without clone/fetch", + ) + + (options, args) = parser.parse_args() + + clone(options) + +def clone(options): + # TODO catch errrors + if not os.path.isdir(options.path): + sys.stderr.write("Error: destination path does not exist " + options.path + "\n") + exit(1) + + git_path = shutil.which("git") + if git_path == "None": + sys.stderr.write("Error: git executable not installed or not in $PATH" + "\n") + exit(2) + else: + print(pname() + " using " + git_path) + + t = time.time() + + gl = gitlab.Gitlab(options.url, options.token) + + group = gl.groups.get(options.namespace, lazy=True, include_subgroups=True) + + projects = [] + + # Get all projects inside the namespace + for project in group.projects.list(all=True): + projects.append(project) + print(pname() + " found " + project.path_with_namespace) + + # Get all projects inside the subgroups + for group in gl.groups.list( + all=True, owned=True, query_parameters={"id": options.namespace} + ): + + for project in group.projects.list(all=True): + projects.append(project) + print(pname() + " found " + project.path_with_namespace) + + subgroups = group.subgroups.list(all=True) + while True: + for subgroup in subgroups: + + real_group = gl.groups.get(subgroup.id, lazy=True) + + for project in real_group.projects.list(all=True): + projects.append(project) + print(pname() + " found " + project.path_with_namespace) + + subgroups = real_group.subgroups.list(all=True) + + if len(subgroups) == 0: next + + if len(subgroups) == 0: break + + if not options.dryrun: + for project in projects: + print(pname() + " clone/fetch project " + project.path_with_namespace) + folders = [f.strip().lower() for f in project.path_with_namespace.split("/")] + if options.noroot: + folders.remove(options.namespace) + + mkdir = options.path + for i in range(len(folders) - 1): + mkdir = mkdir + "/" + folders[i] + if not os.path.isdir(mkdir): + os.mkdir(mkdir) + + clone_path = options.path + "/" + "/".join(str(x) for x in folders) + clone_path = re.sub("/+", "/", clone_path) + print(pname() + " folder " + clone_path) + if not os.path.isdir(clone_path): + print(pname() + " cloning " + project.http_url_to_repo) + try: + subprocess.run(["git", "clone", project.http_url_to_repo, clone_path]) + except: + sys.stderr.write("Unexpected error while cloning: terminating\n") + exit(2) + else: + print(pname() + " fetching " + project.http_url_to_repo) + try: + subprocess.run(["git", "-C", clone_path, "fetch", "--all"]) + except: + sys.stderr.write("Unexpected error while fetching: terminating\n") + exit(3) + + print(pname() + " mission accomplished in " + str(round(time.time() - t, 2)) + "s") + exit(0) + +if __name__ == '__main__': + main() diff --git a/requirements.txt b/requirements.txt index 2239a9e..234a291 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -python-gitlab==2.4.0 +python-gitlab==2.4.0 diff --git a/setup.py b/setup.py index 0953130..1343b35 100644 --- a/setup.py +++ b/setup.py @@ -1,30 +1,30 @@ -from setuptools import setup, find_packages - -with open("README.rst") as readme_file: - readme = readme_file.read() - -requirements = ["python-gitlab"] - -setup( - author="Lucca Pessoa da Silva Matos", - author_email="luccapsm@gmail.com", - description="Command line tool to clone projects inside groups in GitLab", - install_requires=requirements, - license="MIT license", - long_description=readme + "\n\n", - include_package_data=True, - keywords=[ - "gitlab", - "gitlab-clone", - ], - name="gitlab-clone-recurisve", - packages=find_packages(include=["gitlab_clone_recursive"]), - url="https://github.com/lpmatos/gitlab-clone", - version="1.2", - entry_points={ - "console_scripts": [ - "gitlab-clone-recursive=gitlab_clone_recursive.cli:main", - ] - }, - zip_safe=False, -) +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""The setup script.""" + +from setuptools import setup, find_packages + +requirements = ['requests'] + +setup( + author="Lucca Pessoa da Silva Matos", + author_email='luccapsm@gmail.com', + description="Gitlab tool for recursive clone", + install_requires=requirements, + license="MIT license", + include_package_data=True, + keywords=[ + 'gitlab', 'gitlab-api', + ], + name='gitlab-clone-recursive', + packages=find_packages(include=['gitlab']), + url='https://github.com/lpmatos/gitlab-clone-recursive', + version='0.0.2', + entry_points={ + "console_scripts": [ + "gitlab-clone-recursive=gitlab.cli:main", + ] + }, + zip_safe=False, +)