Commit 6b74c2d1 authored by Michael Völske's avatar Michael Völske

clone-groups: Limit thread pools size.

Possibly fixes #10 -- but not definitive, could not be reproduced.
parent 0c0f9c87
......@@ -19,10 +19,12 @@ FLAGS "$@" || exit $?
eval set -- "${FLAGS_ARGV}"
WDIR=/tmp/wcmd-tmp
WVOL=""
[ "${FLAGS_preserve_wdir}" = "${FLAGS_TRUE}" ] && {
mkdir -vp ${FLAGS_wdir}
WDIR=${FLAGS_wdir}
WVOL="-v ${WDIR}:${WDIR}"
}
docker run --rm -it \
......@@ -30,6 +32,7 @@ docker run --rm -it \
-v "${D}:/webis-cmd" \
-v "${HOME}:/wcmd-home" \
-e "PYTHONPATH=/webis-cmd" \
$WVOL \
-e "HOME=/wcmd-home" \
-e "WCMD_TEST_RUNNING=1" \
-e "WCMD_WDIR=${WDIR}" \
......
......@@ -32,13 +32,13 @@ class GroupWithProjects(object):
def gather_groups_and_projects():
gl = get_api_instance()
pool = ThreadPool()
pool = ThreadPool(2)
#import pdb; pdb.set_trace()
groups = pool.apply_async(lambda: gitlab_get('/groups', all_available='true'))
projects = pool.apply_async(lambda: list(tqdm(gl.projects.list(as_list=False, per_page=100))))
groups = {g['path']: GroupWithProjects(g, []) for g in groups.get()}
#import code; code.interact(local=locals())
for p in projects.get():
try:
namespace = p.path_with_namespace.split('/')[0]
......@@ -50,6 +50,7 @@ def gather_groups_and_projects():
groups[namespace].projects.append(p.attributes)
groups = sorted(groups.values(), key=lambda g: g.group['path'])
pool.close()
return groups
......@@ -80,11 +81,14 @@ def clone_all_projects_from_group(group, clone_path, clone_method, user):
subprocess.Popen(['git', 'config', '--local', 'user.name', user['name']], cwd=cwd)
subprocess.Popen(['git', 'config', '--local', 'user.email', user['email']], cwd=cwd)
return retval
pool = ThreadPool()
pool = ThreadPool(8)
# clone projects in parallel
retvals = pool.map(_do_clone, group.projects, 1)
pool.close()
# return failed projects
return [p for p, retval in zip(group.projects, retvals) if retval != 0]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment