implement HTTPS support for cloning (smarkets#225)

marge/app.py

@@ -76,8 +76,14 @@ def regexp(str_regex):
         metavar='URL',
         help='Your GitLab instance, e.g. "https://gitlab.example.com".\n',
     )
-    ssh_key_group = parser.add_mutually_exclusive_group(required=True)
-    ssh_key_group.add_argument(
+    repo_access = parser.add_mutually_exclusive_group(required=True)
+    repo_access.add_argument(
+        '--use-https',
+        env_var='MARGE_USE_HTTPS',
+        action='store_true',
+        help='use HTTPS instead of SSH for GIT repository access\n',
+    )
+    repo_access.add_argument(
         '--ssh-key',
         type=str,
         metavar='KEY',
@@ -87,7 +93,7 @@ def regexp(str_regex):
             'You can still set it via ENV variable or config file, or use "--ssh-key-file" flag.\n'
         ),
     )
-    ssh_key_group.add_argument(
+    repo_access.add_argument(
         '--ssh-key-file',
         type=str,  # because we want a file location, not the content
         metavar='FILE',
@@ -238,7 +244,9 @@ def regexp(str_regex):
 @contextlib.contextmanager
 def _secret_auth_token_and_ssh_key(options):
     auth_token = options.auth_token or options.auth_token_file.readline().strip()
-    if options.ssh_key_file:
+    if options.use_https:
+        yield auth_token, None
+    elif options.ssh_key_file:
         yield auth_token, options.ssh_key_file
     else:
         with tempfile.NamedTemporaryFile(mode='w', prefix='ssh-key-') as tmp_ssh_key_file:
@@ -290,6 +298,8 @@ def main(args=None):
 
         config = bot.BotConfig(
             user=user,
+            use_https=options.use_https,
+            auth_token=auth_token,
             ssh_key_file=ssh_key_file,
             project_regexp=options.project_regexp,
             git_timeout=options.git_timeout,

marge/bot.py

@@ -32,13 +32,22 @@ def __init__(self, *, api, config):
 
     def start(self):
         with TemporaryDirectory() as root_dir:
-            repo_manager = store.RepoManager(
-                user=self.user,
-                root_dir=root_dir,
-                ssh_key_file=self._config.ssh_key_file,
-                timeout=self._config.git_timeout,
-                reference=self._config.git_reference_repo,
-            )
+            if self._config.use_https:
+                repo_manager = store.HttpsRepoManager(
+                    user=self.user,
+                    root_dir=root_dir,
+                    auth_token=self._config.auth_token,
+                    timeout=self._config.git_timeout,
+                    reference=self._config.git_reference_repo,
+                )
+            else:
+                repo_manager = store.SshRepoManager(
+                    user=self.user,
+                    root_dir=root_dir,
+                    ssh_key_file=self._config.ssh_key_file,
+                    timeout=self._config.git_timeout,
+                    reference=self._config.git_reference_repo,
+                )
             self._run(repo_manager)
 
     @property
@@ -186,7 +195,7 @@ def _get_single_job(self, project, merge_request, repo, options):
 
 
 class BotConfig(namedtuple('BotConfig',
-                           'user ssh_key_file project_regexp merge_order merge_opts git_timeout ' +
+                           'user use_https auth_token ssh_key_file project_regexp merge_order merge_opts git_timeout ' +
                            'git_reference_repo branch_regexp source_branch_regexp batch')):
     pass
 

marge/project.py

@@ -77,6 +77,10 @@ def path_with_namespace(self):
     def ssh_url_to_repo(self):
         return self.info['ssh_url_to_repo']
 
+    @property
+    def http_url_to_repo(self):
+        return self.info['http_url_to_repo']
+
     @property
     def merge_requests_enabled(self):
         return self.info['merge_requests_enabled']

marge/store.py

@@ -1,9 +1,10 @@
 import tempfile
+import re
 
 from . import git
 
 
-class RepoManager:
+class SshRepoManager:
 
     def __init__(self, user, root_dir, ssh_key_file=None, timeout=None, reference=None):
         self._root_dir = root_dir
@@ -45,3 +46,49 @@ def root_dir(self):
     @property
     def ssh_key_file(self):
         return self._ssh_key_file
+
+
+class HttpsRepoManager:
+
+    def __init__(self, user, root_dir, auth_token=None, timeout=None, reference=None):
+        self._root_dir = root_dir
+        self._user = user
+        self._auth_token = auth_token
+        self._repos = {}
+        self._timeout = timeout
+        self._reference = reference
+
+    def repo_for_project(self, project):
+        repo = self._repos.get(project.id)
+        if not repo or repo.remote_url != project.http_url_to_repo:
+            credentials = "oauth2:" + self._auth_token
+            # insert token auth "oauth2:<auth_token>@"
+            repo_url = re.sub("(?P<protocol>http(s)?://)", "\g<protocol>" + credentials + "@", project.http_url_to_repo, 1)
+            local_repo_dir = tempfile.mkdtemp(dir=self._root_dir)
+
+            repo = git.Repo(repo_url, local_repo_dir, ssh_key_file=None,
+                               timeout=self._timeout, reference=self._reference)
+            repo.clone()
+            repo.config_user_info(
+                user_email=self._user.email,
+                user_name=self._user.name,
+            )
+
+            self._repos[project.id] = repo
+
+        return repo
+
+    def forget_repo(self, project):
+        self._repos.pop(project.id, None)
+
+    @property
+    def user(self):
+        return self._user
+
+    @property
+    def root_dir(self):
+        return self._root_dir
+
+    @property
+    def auth_token(self):
+        return self._auth_token

tests/test_store.py

@@ -18,7 +18,7 @@ class TestRepoManager:
     def setup_method(self, _method):
         user = marge.user.User(api=None, info=dict(USER_INFO, name='Peter Parker', email='pparker@bugle.com'))
         self.root_dir = tempfile.TemporaryDirectory()
-        self.repo_manager = marge.store.RepoManager(
+        self.repo_manager = marge.store.SshRepoManager(
             user=user, root_dir=self.root_dir.name, ssh_key_file='/ssh/key',
         )