Commit df37a5ba authored by Maik Fröbe's avatar Maik Fröbe

Merge branch 'master' of webis.uni-weimar.de:code-generic/code-webis-cmd

parents ff073aaa b40bccf3
......@@ -10,9 +10,11 @@ betamng Low level scripts for managing Betaweb.
bios_cpu_test_all Starts bios cpu test on all nodes of the hostlist
change_lcd_msg_all Change lcd message of every node of betaweb to betaweb???
check_dimms Check for warnings concerning RAM modules in idrac for all betaweb machines
clear_idrac_log
do_all Run a command sequential on all betamng nodes
enable_pxe_bios_boot_all Enable pxe bios boot on all hosts: experimental! hope and prey
get_bios_boot_settings_all Get boot settings from all nodes
get_idrac_log
pssh Run a command parallel on all betamng nodes
read_all_macs Printout system dump of all hosts for extracting mac addresses
set_critical_temp_shutdown Set settings for shutdown on critical temperature
......@@ -25,6 +27,7 @@ betamng Low level scripts for managing Betaweb.
betaweb High level scripts for betaweb, e.g. hadoop management.
check_ram
htop Print the clipboard to standard output
restart_hadoop_ressourcemanager Restart hadoop ressource manager on betaweb020 and restarts nodemanager
rolling_restart Restart betaweb nodes one after another
rolling_state Apply Salt state on nodes one after another
......@@ -39,6 +42,7 @@ core Core functions of the Webis command.
version Get webis version
cvs Helper scripts for working with the Webis CVS server
onboard Create a new user on the webis virtual machine (webis VM)
status Checks what changed in this CVS directory without doing anything
update Update this CVS directory, deleting directories that were deleted on the server
......@@ -61,16 +65,20 @@ git Helper scripts for working with the Webis Git server and Gi
k8s Kubernetes tools
connect-test-cluster Connect kubectl to the testing cluster
login Set up OpenID authentication for kubernetes access
setup-user-namespaces Set up k8s access per gitlab groups
pki Webis TLS PKI Tools
cert
util General helper tools and utilities
authorized-ssh-keys Collect all SSH keys in use across salt minions' user accounts
backup-btrfs-vol Take a snapshot of a btrfs subvolume and back up its contents to another location
list-buw-dns List all assigned Webis host names / IP addresses at BUW
login Get user name for a university login
parallel-idrac Execute idrac command in parallel across multiple hosts
patch-jar Add files from one jar to another
read-from-clipboard Print the clipboard to standard output
salt-restart restart a salt minion on a targeted machine
sshproxy Set up an SSH proxy tunnel for accessing digital library content or university-restricted URLs
tasks Execute lists of tasks in parallel
webis-web-permissions Find files with missing group write permission on gonzo
......
......@@ -7,6 +7,8 @@
# Project general
# Author: Steve Göring
#
webiscmd_libs_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
webiscmdrootpath="$(${webiscmd_libs_dir}/readlink-f.py $webiscmd_libs_dir/../)/"
#
# Logging macros,
......@@ -81,9 +83,10 @@ check_tools() {
#
read_password() {
__resultvar="$2"
logInfo "$1"
echo -n "$1 "
read -s pw
eval "$__resultvar=\"$pw\""
echo -e '\U2713'
}
#
......@@ -266,6 +269,17 @@ is_var_set() {
[ ! -z "${!1+x}" ]
}
# Check if the git repository in CWD
# has unpulled commits on any remote
# branch, without actually fetching
# anything.
#
git_repo_has_updates() {
git fetch --dry-run --verbose 2>&1 | grep -q 'git-upload-pack\|[0-9a-f]\+\.\.[0-9a-f]\+'
}
#
# Printout settings and check configuration.
#
......
This diff is collapsed.
......@@ -8,6 +8,7 @@ from log import *
GROUP_ID_WEBISSTUD = 117
GROUP_ID_THIRDPARTY = 170
GROUP_ID_AUTH_K8S_USER = 352
try:
## Using Python3
......
......@@ -3,8 +3,7 @@
# based on https://www.cyberciti.biz/faq/howto-move-migrate-user-accounts-old-to-new-server/
# Author: Kai Lorenz
# Last update 07.03.2019
sudo mkdir /move
mkdir /move
cd /home/ || exit
for f in */; do
# do some stuff here with "$f"
......@@ -12,10 +11,9 @@ for f in */; do
varTargerUser="$f"
if [[ $varTargerUser != *"webis"* ]];then
varArchiveName=$({ date +%Y; echo -; hostname; echo -; echo $varTargerUser; } | sed ':a;N;s/\n/ /;ba' | tr -d '[:space:]')
sudo apt-get autoremove -y
echo "creating tar from home"
sudo tar --exclude='/home/$varTargerUser/.local/share/Trash/' --exclude='/home/$varTargerUser/.cache/mozilla' --exclude='/home/$varTargerUser/.wine' --exclude='/home/$varTargerUser/.cache/google-chrome' --exclude='/home/$varTargerUser/.cache/pip' --exclude='/home/$varTargerUser/.cache/thumbnails' -zcvpf /move/$varArchiveName.tar.gz /home/$varTargerUser
sudo mv -uv /move/* /mnt/nfs/webis20/code-in-archive/account-graveyard
apt-get autoremove -y
tar --exclude='/home/$varTargerUser/.local/share/Trash/' --exclude='/home/$varTargerUser/.cache/mozilla' --exclude='/home/$varTargerUser/.wine' --exclude='/home/$varTargerUser/.cache/google-chrome' --exclude='/home/$varTargerUser/.cache/pip' --exclude='/home/$varTargerUser/.cache/thumbnails' -zcvpf /move/$varArchiveName.tar.gz /home/$varTargerUser
mv -uv /move/* /mnt/nfs/webis20/code-in-archive/account-graveyard
fi
done
......@@ -4,6 +4,11 @@
# Author: Kai Lorenz
# Last update 24.02.2019
if (( EUID != 0 )); then
echo "Please execute this script as root (sudo su)." 1>&2
exit 1
fi
ls /home/
echo "Please enter the target username"
read varTargerUser
......@@ -12,9 +17,5 @@ echo # (optional) move to a new line
if [[ $REPLY =~ ^[Yy]$ ]]
then
varArchiveName=$({ date +%Y; echo -; hostname; echo -; echo $varTargerUser; } | sed ':a;N;s/\n/ /;ba' | tr -d '[:space:]')
sudo mkdir /move
sudo apt-get autoremove -y
echo "creating tar from home"
sudo tar --exclude='/home/$varTargerUser/.local/share/Trash/' --exclude='/home/$varTargerUser/.cache/mozilla' --exclude='/home/$varTargerUser/.wine' --exclude='/home/$varTargerUser/.cache/google-chrome' --exclude='/home/$varTargerUser/.cache/pip' --exclude='/home/$varTargerUser/.cache/thumbnails' -zcvpf /move/$varArchiveName.tar.gz /home/$varTargerUser
sudo mv -uv /move/* /mnt/nfs/webis20/code-in-archive/account-graveyard
tar --exclude='/home/$varTargerUser/.local/share/Trash/' --exclude='/home/$varTargerUser/.cache/mozilla' --exclude='/home/$varTargerUser/.wine' --exclude='/home/$varTargerUser/.cache/google-chrome' --exclude='/home/$varTargerUser/.cache/pip' --exclude='/home/$varTargerUser/.cache/thumbnails' -zcvpf /mnt/nfs/webis20/code-in-archive/account-graveyard/$varArchiveName.tar.gz /home/$varTargerUser
fi
#!/usr/bin/env bash
#
# Author: Martin Heinrich
#
#
#
# clear event log of idrac
# it uses the PowerEdge Healt CRIT Error in check_mk
#
scriptPath=${0%/*}
. "$scriptPath"/../../libs/bashhelper.sh
. "$scriptPath"/../../libs/shflags
usage() {
echo "
Usage:
$(basename "$0")
Description:
clear event log of idrac
it uses the PowerEdge Healt CRIT Error in check_mk
Examples:
$(basename "$0")
"
exit 1
}
FLAGS_HELP=$(usage)
export FLAGS_HELP
FLAGS "$@" || exit 1 # Parse command line arguments.
eval set -- "${FLAGS_ARGV}"
main() {
logInfo "get idrac log"
filename="/tmp/powerEdge-hostslist.csv"
curl -o ${filename} "https://webis111.medien.uni-weimar.de/monitoring/check_mk/view.py?view_name=powerEdge_Health&_username=web_auto&_secret=XHFESMBBPVUAAPAJTVHU&display_options=Tbfcdexilwoz&output_format=csv"
# delete first line in CSV File
# delete " characters
tail -n +2 "${filename}" | sed 's/"//g' > "${filename}.tmp" && mv "${filename}.tmp" "${filename}"
# --par maximum number of parallel threads
$(dirname $0)/pssh/bin/pssh --inline --askpass --user=webis --hosts="${filename}" \
--outdir=EventlogOut --errdir=err \
--par=10 \
--option="StrictHostKeyChecking=no" "racadm clrsel"
rm "${filename}"
}
#
# Start program with parameters
#
main "$@"
#!/usr/bin/env bash
#
# Author: Martin Heinrich
#
#
#
# printout event log of idrac
# it uses the PowerEdge Healt CRIT Error in check_mk
#
scriptPath=${0%/*}
. "$scriptPath"/../../libs/bashhelper.sh
. "$scriptPath"/../../libs/shflags
usage() {
echo "
Usage:
$(basename "$0")
Description:
printout event log of idrac
it uses the PowerEdge Healt CRIT Error in check_mk
Examples:
$(basename "$0")
"
exit 1
}
FLAGS_HELP=$(usage)
export FLAGS_HELP
FLAGS "$@" || exit 1 # Parse command line arguments.
eval set -- "${FLAGS_ARGV}"
# check requirement
command -v curl >/dev/null 2>&1 || { echo "curl is required, but not installed. please run apt install curl" >&2; exit 1; }
main() {
logInfo "get idrac log"
filename="/tmp/powerEdge-hostslist.csv"
curl -o ${filename} "https://webis111.medien.uni-weimar.de/monitoring/check_mk/view.py?view_name=powerEdge_Health&_username=web_auto&_secret=XHFESMBBPVUAAPAJTVHU&display_options=Tbfcdexilwoz&output_format=csv"
# delete first line in CSV File
# delete " characters
tail -n +2 "${filename}" | sed 's/"//g' > "${filename}.tmp" && mv "${filename}.tmp" "${filename}"
# --par maximum number of parallel threads
$(dirname $0)/pssh/bin/pssh --inline --askpass --user=webis --hosts="${filename}" \
--outdir=EventlogOut --errdir=err \
--par=10 \
--option="StrictHostKeyChecking=no" "racadm getsel"
rm "${filename}"
}
#
# Start program with parameters
#
main "$@"
......@@ -4,12 +4,12 @@
# Copyright 2014-today www.webis.de
#
# Project WEBIS
# Author: Steve Göring
# Author: Michael Völske
#
scriptPath=${0%/*}
. "$scriptPath"/../../libs/bashhelper.sh
check_tools "ssh"
check_tools "sshpass" "ssh" "xargs"
#
# Define usage screen.
......@@ -26,7 +26,11 @@ Usage:
#
#
main() {
"$scriptPath/pssh.sh" "racadm serveraction powerstatus"
read_password "Enter betamng password:" SSHPASS
export SSHPASS
seq -f "betamng%03.0f" 1 135 \
| webis util parallel-idrac --cmd 'racadm serveraction powerstatus'
}
#
......
#!/bin/bash
# Print the clipboard to standard output
#
# Copyright 2019-today
#
# Project WEBIS
# Author: Johannes Kiesel
scriptPath=${0%/*}
. "$scriptPath"/../../libs/bashhelper.sh
. "$scriptPath"/../../libs/shflags
check_tools "cssh"
#
# Define usage screen.
#
usage() {
echo "usage:
$(basename "$0")
description:
open htop on all machines of betaweb.
"
}
#
# Define command line arguments and parse them.
#
FLAGS_HELP=$(usage)
export FLAGS_HELP
FLAGS "$@" || exit 1 # Parse command line arguments.
eval set -- "${FLAGS_ARGV}"
#
# Main
#
main() {
cssh -o "-t" -a 'htop' betaweb{001,002,003,004,005,006,007,008,009,010,011,012,013,014,015,016,017,018,019,021,022,023,024,025,026,027,028,029,030,031,032,033,034,035,036,037,038,039,040,041,042,043,044,045,046,047,048,049,050,051,052,053,054,055,056,057,058,059,060,061,062,063,064,065,066,067,068,069,070,071,072,073,074,075,076,077,078,079,080,081,082,083,084,085,086,087,088,089,090,091,092,093,094,095,096,097,098,099,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130}.medien.uni-weimar.de
}
#
# Start programm with parameters.
#
main "$@"
#!/bin/bash
# Update the webis command from Git
#
# Update Webis command
#
# Copyright 2015-today
#
# Project WEBIS
# Author: Steve Göring
#
# Load libaries and toolkits.
#
scriptPath=${0%/*}
scriptPath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
. ${scriptPath}/../../libs/bashhelper.sh
. ${scriptPath}/../../libs/shflags
......@@ -46,24 +39,22 @@ eval set -- "${FLAGS_ARGV}"
#
#
main() {
webiscmdrootpath="$(${scriptPath}/../../libs/readlink-f.py $scriptPath/../../)/"
logInfo "${scriptPath}"
cd "$webiscmdrootpath"
logInfo "Checking for updates... ${webiscmdrootpath}"
LOCAL=$(git rev-parse '@{0}')
REMOTE=$(git ls-remote origin HEAD | cut -f1)
BASE=$(git merge-base '@{0}' '@{u}')
if [ "$LOCAL" = "$REMOTE" ] || [ "$REMOTE" = "$BASE" ]; then
logInfo "No update needed."
else
if git_repo_has_updates; then
logInfo "Updating webis-cmd."
git pull --rebase origin master
local cmd="git"
if [ ! -w "$(pwd)" ]; then
cmd="sudo git"
fi
$cmd pull --rebase origin master
else
logInfo "No update needed."
fi
}
#
# Start programm with parameters.
#
set -e
main "$@"
#!/bin/bash
# Create a new user on the webis virtual machine (webis VM)
#
# Copyright 2019-today
#
# Project WEBIS
# Author: Johannes Kiesel
#
# Install.
#
# On webis.uni-weimar.de:
# - sudo visudo
#
# # Allow members of group webis to run /usr/bin/webisuseradd
# %webis ALL=/usr/bin/webisuseradd
#
# - sudo vim /usr/bin/webisuseradd
#
# #!/bin/bash
# newlogin=$1
#
# if [ -z "$newlogin" ];then
# echo "Usage: $0 <newlogin>"
# exit 1
# fi
#
# useradd -g webis -G webisstud -s /bin/bash -m -d /home/"$newlogin" "$newlogin"
# passwd "$newlogin"
# chage -d 0 "$newlogin"
#
# - sudo chmod +x /usr/bin/webisuseradd
#
#
# Load libaries and toolkits.
#
scriptPath=${0%/*}
. "$scriptPath"/../../libs/bashhelper.sh
. "$scriptPath"/../../libs/shflags
check_tools "ssh"
#
# Define usage screen.
#
usage() {
echo "
usage:
$(basename "$0") newlogin [yourlogin]
description:
create a new webis staff user on the webis virtual machine
requires your login account on webis.uni-weimar.de to be in the webis group
examples:
$(basename "$0") stein
"
exit 1
}
#
# Define command line arguments and parse them.
#
FLAGS_HELP=$(usage)
export FLAGS_HELP
FLAGS "$@" || exit 1 # Parse command line arguments.
eval set -- "${FLAGS_ARGV}"
#
#
#
main() {
if [ "$#" -eq 0 ]; then
logError "Missing arguments see:"
usage
fi
logInfo "creating $1"
user=$2
if [ -z "$user" ];then
user="$USER"
fi
logInfo "Note you may have to enter your password twice, as the account creation works by sudo"
logInfo "Now connecting to webis.uni-weimar.de as $user"
name="$1"
ssh $user@webis.uni-weimar.de -t "sudo /usr/bin/webisuseradd \"$name\""
}
#
# Start programm with parameters.
#
main "$@"
......@@ -41,7 +41,7 @@ NO_COLOR='\033[0m'
# Main
#
main() {
local tmp=/tmp/cvsupdate-$$
local tmp=~/tmp/cvsupdate-$$
echo "CVS STATUS"
webis cvs status 2> /dev/zero | grep "Needs merge" | awk '{print " - "$NF}' > $tmp
if [ $(cat $tmp | wc -l) -gt 0 ];then
......
#!/bin/sh
# Set up k8s access per gitlab groups
"true" '''\'
# try running as python3, if that fails fall back to (any) python
command -v python3 > /dev/null
if [ $? -eq 0 ]; then
exec env python3 "$0" "$@"
fi
command -v python > /dev/null
if [ $? -eq 0 ]; then
exec env python "$0" "$@"
else
echo -e "\033[91m[ERROR] Install Python and try again!" 1>&2
exit 1
fi
'''
from __future__ import print_function
import os
import shutil
import sys
import tempfile
import loader
import gitlab
from log import *
from lib import get_selection_from_list, confirm_prompt
from webis_gitlab import get_api_instance, GROUP_ID_AUTH_K8S_USER
template = """
apiVersion: v1
kind: Namespace
metadata:
name: {username}
labels:
source: webiscmd
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
namespace: {username}
name: {username}
labels:
source: webiscmd
rules:
- apiGroups: ["*"]
resources: ["*"]
verbs: ["*"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {username}-binding
namespace: {username}
labels:
source: webiscmd
subjects:
- kind: User
name: oidc:{email}
apiGroup: rbac.authorization.k8s.io
roleRef:
kind: Role
name: {username}
apiGroup: rbac.authorization.k8s.io"""
gl = get_api_instance()
group = gl.groups.get(GROUP_ID_AUTH_K8S_USER)
users = (dict(username=u.attributes['username'],email=gl.users.get(u.attributes['id']).attributes['email']) for u in group.members.list(all=True))
tmp = tempfile.mkdtemp()
roles_file = os.path.join(tmp, 'roles.yaml')
with open(roles_file, 'w') as f:
f.write("\n---\n".join(template.format(**u) for u in users))
lInfo("Deleting existing k8s-auth/user resources")
for obj in ['rolebinding', 'role']:
os.system('kubectl delete %s -l source==webiscmd -A' % obj)
lInfo("Creating up-to-date k8s-auth/user resources from gitlab group membership")
os.system('kubectl replace -f ' + roles_file)
shutil.rmtree(tmp)
This diff is collapsed.
......@@ -75,6 +75,7 @@ policy = policy_match
name_opt = ca_default
cert_opt = ca_default
unique_subject = no
copy_extensions = copy
[ policy_match ]
commonName = supplied
......
#!/bin/bash
# Collect all SSH keys in use across salt minions' user accounts
#
# Copyright 2019-today
#
# Project WEBIS
# Author: Michael Völske
scriptPath=${0%/*}
. "$scriptPath"/../../libs/bashhelper.sh
. "$scriptPath"/../../libs/shflags
check_tools "ssh" "salt"
#
# Define usage screen.
#
usage() {
echo "usage:
$(basename "$0")
description:
Collect all SSH keys occurring in .ssh/authorized_keys files across all reachable salt minions.
Must be run from a salt master.
"
}
#
# Define command line arguments and parse them.
#
FLAGS_HELP=$(usage)
export FLAGS_HELP
FLAGS "$@" || exit 1 # Parse command line arguments.
eval set -- "${FLAGS_ARGV}"
#
# Main
#
main() {
NUM_DEAD=0
KEYCOUNT=0
declare -A KPH
LIST=$( mktemp )
salt '*' cmd.run 'find /root /home -path "*/.ssh/authorized_keys" -maxdepth 4 -exec cat \{\} \;' --output=txt \
| sort -t: -k2 -k1V \
> $LIST
while IFS=":" read -r -a LINE ; do
H=${LINE[0]}
K=${LINE[1]}
if [[ $K =~ "did not return" ]]; then
NUM_DEAD=$(( NUM_DEAD + 1))
continue
fi
if [[ $K =~ ^[\ \t]*$ ]]; then K=""; fi
if [[ $K != $LASTKEY && $K != "" ]]; then
echo Key: $K
echo Used on hosts:
KEYCOUNT=$(( $KEYCOUNT + 1 ))
LASTHOST=""
fi
if [[ $K != "" && $H != $LASTHOST ]]; then
echo " - $H"
LASTHOST=$H
KPH[$H]=$(( ${KPH[$H]} + 1 ))
fi
LASTKEY=$K
done < $LIST
rm -f $LIST
echo "TOTAL UNIQUE KEYS: $KEYCOUNT; $NUM_DEAD hosts offline; ${#KPH[@]} hosts online."
}
#
# Start programm with parameters.
#
main "$@"
......@@ -96,9 +96,10 @@ main() {
set -e
btrfs_snapshot "$FLAGS_volume" "$FLAGS_snapshot"
run_backup "$FLAGS_snapshot" $@
cleanup() { snapshot_delete "$FLAGS_snapshot"; }
trap cleanup EXIT
snapshot_delete "$FLAGS_snapshot"
run_backup "$FLAGS_snapshot" $@
}
......
#!/bin/bash
# Execute idrac command in parallel across multiple hosts
#
# Copyright 2014-today www.webis.de
#
# Project WEBIS
# Author: Michael Völske
#
scriptPath=${0%/*}
. "$scriptPath"/../../libs/bashhelper.sh
. "$scriptPath"/../../libs/shflags
check_tools "sshpass" "ssh" "xargs"
usage() {
echo "
Usage:
$(basename "$0")
Description:
Execute a given idrac command across multiple hosts and return the output, optionally processed through a filter.
A list of hostnames should be passed on stdin, one per line. The environment variable SSHPASS must be set to the iDrac password.
"
exit 0
}
DEFINE_string 'cmd' 'racadm serveraction powerstatus' 'iDrac command to execute' 'c'
DEFINE_string 'user' 'webis' 'iDrac username' 'u'
DEFINE_string 'filtercmd' 'cat' 'filter (command) to pass Idrac output from each individual host through' 'f'
DEFINE_string 'sortcmd' 'sort' 'Command to sort all output lines at the end' 's'
export FLAGS_HELP=$(usage)
FLAGS "$@" || exit 0
eval set -- "${FLAGS_ARGV}"
main() {
xargs -n1 -P0 -I% bash -c \
"echo -e % : \$( \
sshpass -e ssh \
-o StrictHostkeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=3s \
${FLAGS_user}@% \
${FLAGS_cmd} \
2>/dev/null \
| ${FLAGS_filtercmd} \
) " \
| ${FLAGS_sortcmd}
}
main "$@"
#!/bin/bash
# restart a salt minion on a targeted machine
ssh -t webis@$1 'sudo service salt-minion restart'
echo done
\ No newline at end of file
#!/bin/sh
#!/bin/bash