Commit 80f42891 authored by Michael Völske's avatar Michael Völske

Summary: git/clone-groups: add support for Python2 andHTTPS auth

parent 7b3156c4
#!/usr/bin/env python
# Check out groups of webis gitlab projects at once
from __future__ import print_function
import configparser
import json
import os
import re
import subprocess
import sys
import urllib
import urllib.parse
import urllib.request
## Using Python3
import configparser
from urllib.request import urlopen
from urllib.parse import urlencode
def get_headers_and_json_response(url):
with urlopen(url) as response:
data = json.loads('utf-8'))
headers = response.getheaders()
headers = dict((k.lower(), v) for k, v in headers)
return (headers, data)
except ImportError:
## Using Python2
import ConfigParser as configparser
from urllib import urlopen
from urllib import urlencode
def get_headers_and_json_response(url):
response = urlopen(url)
return (response.headers.dict, json.loads(
input = raw_input
config = configparser.ConfigParser()["HOME"], ".python-gitlab.cfg"))
......@@ -18,24 +38,23 @@["HOME"], ".python-gitlab.cfg"))
def custom_get_request(path, **params):
url = config['webis']['url']
params['private_token'] = config['webis']['private_token']
url = config.get('webis', 'url')
params['private_token'] = config.get('webis', 'private_token')
except KeyError:
sys.stderr.write("Error reading config file. Have you run 'webis git init'?\n")
url = "{}/api/v3/{}?{}".format(url, path, urllib.parse.urlencode(params))
with urllib.request.urlopen(url) as response:
data ='utf-8')
return (dict(response.getheaders()), json.loads(data))
url = "{}/api/v3/{}?{}".format(url, path, urlencode(params))
headers, data = get_headers_and_json_response(url)
return (headers, data)
def custom_get_all_pages(path, **params):
params['per_page'] = 100
params['page'] = 1
headers, data = custom_get_request(path, **params)
while ('X-Page' in headers and
headers['X-Page'] != headers['X-Total-Pages']):
params['page'] = int(headers['X-Page']) + 1
while ('x-page' in headers and
int(headers['x-page']) != int(headers['x-total-pages'])):
params['page'] = int(headers['x-page']) + 1
headers, data1 = custom_get_request(path, **params)
data += data1
return data
......@@ -68,7 +87,7 @@ def get_group_selections(groups):
groups_with_num = dict(enumerate(groups, 1))
for n, g in sorted(groups_with_num.items()):
print(n, '\t', g)
selected = input('Select the group number(s) to clone, * for everything:')
selected = input('Select the group number(s) to clone, * for everything: ')
if '*' in selected:
selected = set(groups_with_num.keys())
......@@ -78,31 +97,64 @@ def get_group_selections(groups):
return selected
def clone_all_projects_from_group(group, clone_path):
def clone_all_projects_from_group(group, clone_path, clone_method):
for p in group.projects:
pid = p['id']
git_url = p['ssh_url_to_repo']
retval =
['git', 'clone', git_url, os.path.join(clone_path, p['name'])])
retval = clone_method.clone_project(p, clone_path)
if retval != 0:
yield p
def clone_selected_groups(selected):
def clone_selected_groups(selected, clone_method):
failed = []
for g in selected:
failed_this_group = list(clone_all_projects_from_group(g,['path']))
failed_this_group = list(clone_all_projects_from_group(g,['path'], clone_method))
if len(failed_this_group) > 0:
failed.append((g, failed_this_group))
return failed
class CloneMethod(object):
def __init__(self, method):
if 'http' in method:
print('Using HTTPS clone method')
self.key = 'http_url_to_repo'
self.username = input("Enter your user name: ")['git', 'config', '--global',
'credential.helper', 'cache --timeout=7200'])
self.https = True
print("Using SSH clone method")
self.key = 'ssh_url_to_repo'
self.https = False
def clone_project(self, project, clone_path):
url = project[self.key]
if self.https:
url = url.replace("https://", "https://{}@".format(self.username))
['git', 'clone', url, os.path.join(clone_path, project['name'])])
def select_clone_method():
print("Select your method for cloning webis Gitlab repositories: ")
print(" 1. HTTPS (need username and password)")
print(" 2. SSH (need to have an SSH key set up in Gitlab)")
n = input("Your selection: ")
while re.match(r'\s*[12]\s*', n) is None:
print("Invalid selection:", n, "Choose 1 or 2.")
n = input("Your selection: ")
n = int(n.strip())
return CloneMethod('http') if n == 1 else CloneMethod('ssh')
if __name__ == '__main__':
method = select_clone_method()
print("Downloading project information. Please wait...")
groups = gather_groups_and_projects()
selected = get_group_selections(groups)
failed = clone_selected_groups(selected)
failed = clone_selected_groups(selected, method)
if len(failed) > 0:
sys.stderr.write('The following projects could not be cloned successfully:\n')
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment