Commit dde30e3c authored by Michael Völske's avatar Michael Völske

clone-groups: add threading & progress bar

parent 3af1daa1
File added
......@@ -9,12 +9,15 @@ import subprocess
import sys
from argparse import ArgumentParser
# GIL does not apply to parallelising web requests and git-clones
from multiprocessing.dummy import Pool as ThreadPool
import loader
from six.moves import input
from lib import get_selection_from_list
from webis_gitlab import gitlab_config, gitlab_get, get_api_instance
from log import lWarn
from six.moves import input
from tqdm import tqdm
from webis_gitlab import gitlab_config, gitlab_get, get_api_instance
class GroupWithProjects(object):
......@@ -28,13 +31,16 @@ class GroupWithProjects(object):
def gather_groups_and_projects():
gl = get_api_instance()
groups = gitlab_get('/groups', all_available='true')
projects = gl.projects.list(as_list=False)
groups = {g['path']: GroupWithProjects(g, []) for g in groups}
for p in projects:
pool = ThreadPool()
groups = pool.apply_async(lambda: gitlab_get('/groups', all_available='true'))
projects = pool.apply_async(lambda: list(tqdm(gl.projects.list(as_list=False, per_page=100))))
groups = {g['path']: GroupWithProjects(g, []) for g in groups.get()}
#import code; code.interact(local=locals())
for p in projects.get():
#import code; code.interact(local=locals())
namespace = p.path_with_namespace.split('/')[0]
except TypeError as e:
lWarn("Invalid project {}".format(p))
......@@ -67,14 +73,20 @@ def get_group_selections(groups):
def clone_all_projects_from_group(group, clone_path, clone_method, user):
for p in group.projects:
def _do_clone(p):
retval = clone_method.clone_project(p, clone_path)
if retval != 0:
yield p
if retval == 0:
cwd = os.path.join(clone_path, p['name'])
subprocess.Popen(['git', 'config', '--local', '', user['name']], cwd=cwd)
subprocess.Popen(['git', 'config', '--local', '', user['email']], cwd=cwd)
return retval
pool = ThreadPool()
# clone projects in parallel
retvals =, group.projects, 1)
# return failed projects
return [p for p, retval in zip(group.projects, retvals) if retval != 0]
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment