initial commit of WEBIS cmd wrapper

.cvsignore

+__pycache__

config.json

+{
+    "moduls_directory": "tools/",
+    "allowed_scripts": ["python", "bash"],
+    "ignored_dirs": ["CVS"]
+}
\ No newline at end of file

libs/.cvsignore

+__pycache__

libs/bashconventions

+#!/usr/bin/env python3
+#
+#   bash code convention checker
+#
+#   author: Steve Göring, 2015
+#
+import sys
+import os
+import re
+
+def colorred(m):
+    return "\033[91m" + m + "\033[0m"
+
+
+def colorblue(m):
+    return "\033[94m" + m + "\033[0m"
+
+
+def colorgreen(m):
+    return "\033[92m" + m + "\033[0m"
+
+
+def colorcyan(m):
+    return "\033[96m" + m + "\033[0m"
+
+
+def logInfo(msg):
+    print(colorgreen("[INFO ] ") + str(msg))
+
+
+def logError(msg):
+    print(colorred("[ERROR] ") + str(msg))
+
+
+def logDebug(msg):
+    print(colorblue("[DEBUG] ") + str(msg))
+
+
+def logWarn(msg):
+    print(colorcyan("[WARN ] ") + str(msg))
+
+
+def checkFile(filename):
+    logInfo("handle file: " + filename)
+    f = open(filename, "r")
+    linenumber = 1
+
+    # better rules map
+    project = False
+    copyright = False
+    usageDef = False
+    mainDef = False
+    mainCall = False
+    tabsUsed = False
+    multi4SpacesIdent = True
+    commentStartWithSpace = True
+    author = False
+    correctIf = True
+    openIf = False
+    correctCase = True
+    openCase = False
+    correctFor = True
+    openForLoop = False
+    correctWhile = True
+    openWhileLoop = False
+    correctBashSubCall = True
+    correctFunctionDef = True
+
+    errorcount = 0
+    lineNumberMap = {}
+    for l in f:
+        l = l.replace("\n","")
+        # each file starts with #!/bin/bash
+        if linenumber == 1:
+            if  l != "#!/bin/bash":
+                logError("file have to start with #!/bin/bash, fix it!")
+                return
+            linenumber += 1
+            continue
+
+        if "#" in l and "project" in l.lower():
+            project = True
+
+        if "#" in l and "copyright" in l.lower():
+            copyright = True
+        if "#" in l and "author" in l.lower():
+            author = True
+        if "usage()" in l:
+            usageDef = True
+        if "main()" in l:
+            mainDef = True
+        if 'main "$@"' in l:
+            mainCall = True
+        if "\t" in l:
+            tabsUsed = True
+            errorcount += 1
+            lineNumberMap["tabs"] = lineNumberMap.get("tabs",[]) + [linenumber]
+
+        if len(l) != 0 and l[0] == " ":
+            countleadingspaces = 0
+            for i in range(len(l)):
+                if l[i] == " ":
+                    countleadingspaces +=1
+                else:
+                    break
+            if countleadingspaces % 4 != 0:
+                multi4SpacesIdent = False
+                errorcount += 1
+                lineNumberMap["4spaces"] = lineNumberMap.get("4spaces",[]) + [linenumber]
+
+        if "#" in l and l[-1] != "#" and l[l.find("#") + 1] != " " and "$#" not in l and \
+            (re.match(".*\".*#.*\".*",l) == None) and \
+            (re.match(".*'.*#.*'.*",l) == None) :
+            commentStartWithSpace = False
+            errorcount += 1
+            lineNumberMap["commentStartWithSpace"] = lineNumberMap.get("commentStartWithSpace",[]) + [linenumber]
+
+        # if statement
+        if "if" in l and (not "#" in l or l.find("#") > l.find("if") ) and \
+            ("$if" not in l) and \
+            (re.match(".*if[a-z,A-z,0-9]*.*", l) == None) and \
+            (re.match("if \[.*\]; then",l.strip()) == None) and \
+            (re.match(".*$(.*if.*).*",l) == None ) and \
+            (re.match(".*\".*if.*\".*",l) == None ) and \
+            (re.match(".*'.*if.*'.*",l) == None ):
+            correctIf = False
+            errorcount += 1
+            lineNumberMap["if"] = lineNumberMap.get("if",[]) + [linenumber]
+            openIf = True
+        if openIf and "else" in l and l.strip() != "else":
+            correctIf = False
+            errorcount += 1
+            lineNumberMap["if"] = lineNumberMap.get("if",[]) + [linenumber]
+        if openIf and "fi" in l and re.match(".*\".*fi.*", l) == None:
+            openIf = False
+            if len(l.strip()) != len(l):
+                correctIf = False
+                errorcount += 1
+                lineNumberMap["if"] = lineNumberMap.get("if",[]) + [linenumber]
+
+        # case statement
+        if "case" in l and (not "#" in l or l.find("#") > l.find("case") ) and re.match("case .* in", l.strip()) == None:
+            correctCase = False
+            errorcount += 1
+            lineNumberMap["case"] = lineNumberMap.get("case",[]) + [linenumber]
+            openCase = True
+        if openCase and "esac" in l:
+            openCase = False
+            if l.strip() != "esac":
+                correctCase = False
+                lineNumberMap["case"] = lineNumberMap.get("case",[]) + [linenumber]
+                errorcount += 1
+
+        # for loop
+        if "for" in l and re.match(".*\".*for.*", l) == None and (not "#" in l or l.find("#") > l.find("for") ) and re.match("for .* in .*; do", l.strip()) == None and \
+            (re.match(".*for[a-z,A-z,0-9]*.*", l) == None) :
+            correctFor = False
+            lineNumberMap["for"] = lineNumberMap.get("for",[]) + [linenumber]
+            openForLoop = True
+            errorcount += 1
+
+        if openForLoop and "done" in l:
+            openForLoop = False
+            if l.strip() != "done":
+                correctFor = False
+                errorcount += 1
+                lineNumberMap["for"] = lineNumberMap.get("for",[]) + [linenumber]
+
+        # while loop
+        if "while" in l and (not "#" in l or l.find("#") > l.find("while") ) and\
+            re.match("while \[ .* \]; do", l.strip()) == None and\
+            re.match(".*while .*; do", l.strip()) == None:
+            correctWhile = False
+            errorcount += 1
+            lineNumberMap["while"] = lineNumberMap.get("while",[]) + [linenumber]
+            openWhileLoop = True
+        if openWhileLoop and "done" in l:
+            openWhileLoop = False
+            if l.strip() != "done":
+                correctWhile = False
+                lineNumberMap["while"] = lineNumberMap.get("while",[]) + [linenumber]
+                errorcount += 1
+        # bash sub call
+        if "`" in l or "´" in l:
+            correctBashSubCall = False
+            lineNumberMap["bashsubcall"] = lineNumberMap.get("bashsubcall",[]) + [linenumber]
+            errorcount += 1
+
+        # function definitions:
+        if re.match("function .*()", l.strip()) != None:
+            correctFunctionDef = False
+            lineNumberMap["funcdef"] = lineNumberMap.get("funcdef",[]) + [linenumber]
+            errorcount +=1
+
+        linenumber +=1
+
+    if not project:
+        logError("file needs a project description")
+        errorcount += 1
+    if not copyright:
+        logError("file needs a copyright description")
+        errorcount += 1
+    if not usageDef:
+        logWarn("file should have a usage function")
+    if not mainDef:
+        logWarn("file should have a main method")
+        errorcount += 1
+    if mainDef and not mainCall:
+        logError("file needs a correct main function call: e.g. main \"$@\"")
+        errorcount += 1
+    if not author:
+        logError("file needs an author")
+        errorcount += 1
+
+    if tabsUsed:
+        logError("identation with tabs detected, uses multiple of 4 spaces! " + str(lineNumberMap["tabs"]))
+    if not multi4SpacesIdent:
+        logError("identation have to be multiple of 4 spaces: lines: " + str(lineNumberMap["4spaces"]))
+    if not commentStartWithSpace:
+        logError("comment need space after hashtag, not: '#Comment', better: '# Comment', lines:" + str(lineNumberMap["commentStartWithSpace"]))
+    if not correctIf:
+        logError("if statement not correct, should be 'if [ .. ]; then'" + str(lineNumberMap["if"]))
+    if not correctCase:
+        logError("case statement not correct, should be 'case ... in'" + str(lineNumberMap["case"]))
+    if not correctFor:
+        logError("for statement not correct, should be 'for .. in ..; do'" + str(lineNumberMap["for"]))
+    if not correctWhile:
+        logError("while statement not correct, should be while [ .. ]; do" + str(lineNumberMap["while"]))
+    if not correctBashSubCall:
+        logError("dont use `prog`, better use $(prog)" + str(lineNumberMap["bashsubcall"]))
+    if not correctFunctionDef:
+        logError("invalid function definition, should be 'functionname() {'" + str(lineNumberMap["funcdef"]))
+
+    if errorcount != 0:
+        logWarn(str(errorcount) + " errors detected")
+        sys.exit(-1)
+    else:
+        logInfo("everything is ok")
+    f.close()
+
+    return
+
+
+def main(params):
+    usage = "usage: %prog bashShellScript"
+    if(len(params) == 0):
+        logError("need inputfile: " + usage)
+        exit(1)
+    files = params
+
+    for f in files:
+        checkFile(f)
+
+if __name__ == "__main__":
+    main(sys.argv[1:])
\ No newline at end of file

libs/bashhelper.sh

+#!/bin/bash
+#
+#    Copyright 2015-today www.webis.de
+#
+#    Bashhelper functions.
+#
+#    Project general
+#    Author: Steve Göring
+#
+
+#
+#    Logging macros,
+#        e.g. for errors, warnings, information, debug, ...
+#
+
+#
+#    Printout error to stderr.
+#
+logError() {
+    echo -e "\033[91m[ERROR]\033[0m $@ " 1>&2;
+}
+
+#
+#    Printout info message.
+#
+logInfo() {
+    echo -e "\033[92m[INFO ]\033[0m $@"
+}
+
+#
+#    Printout debug info if debug enabled.
+#
+logDebug() {
+    echo -e "\033[94m[DEBUG]\033[0m $@" 1>&2;
+}
+
+#
+#    Printout warning.
+#
+logWarn() {
+    echo -e "\033[96m[WARN ]\033[0m $@" 1>&2;
+}
+
+#
+#    Printout todo things.
+#
+logTodo() {
+    echo -e "\033[35m[TODO ]\033[0m $@" 1>&2;
+}
+
+logCall() {
+    echo "$(date) : $@" >> "$_LOG_FILE"
+}
+
+#
+#    Checks if needed tools are available.
+#
+#    \params $@ list of tools
+#    Example call:
+#        checktools "bash nano"
+#
+check_tools() {
+    for tool in $@; do
+        which $tool > /dev/null
+
+        if [[ "$?" -ne 0 ]]; then
+            logError "$tool is not installed."
+            exit 1
+        fi
+    done
+    logDebug "Each needed tool ($@) is installed."
+}
+
+#
+#    Reads a password from stdin.
+#
+#    \params $1 promt message
+#    \params $2 name of result variable
+#    Example call:
+#        read_password "Please insert your secret password ;)" passwordvar
+#
+read_password() {
+    __resultvar="$2"
+    logInfo "$1"
+    read -s pw
+    eval "$__resultvar=\"$pw\""
+}
+
+#
+#    Reads (y)es or (n) from stdin, exit if 'y' was not choosen.
+#
+#    \params $1 promt message
+#    \params $2 message for "no" case
+#    Example call:
+#        yes_no_promt "Is bash not wunderful?"
+#
+yes_no_promt() {
+    logInfo "$1 [y|n]: "
+    read yesno
+
+    if [ "$yesno" = n ]; then
+        logInfo "$2"
+        exit 1
+    fi
+
+    if [ "$yesno" != n ] && [ "$yesno" != y ]; then
+        logError "Error, usage: [y|n]"
+        exit 1
+    fi
+}
+
+#
+#    SSH key exchange.
+#
+#    \params $1 user at host
+#    \params $2 user password
+#    Example call:
+#        ssh_key_exchange "myuser@mypc" "mysecretpassword"
+#
+ssh_key_exchange() {
+    userathost="$1"
+    pw="$2"
+    # create a ssh key if there is no one
+    if [ ! -f ~/.ssh/id_rsa ]; then
+        logInfo "Rsa key does not exist, create one."
+        ssh-keygen -N "" -f ~/.ssh/id_rsa
+    fi
+    cat ~/.ssh/id_rsa.pub | sshpass -p "$pw" ssh "$userathost" \
+        -o UserKnownHostsFile=/dev/null \
+        -o StrictHostKeyChecking=no \
+        -o LogLevel=error \
+        "(cat > tmp.pubkey ; \
+            mkdir -p .ssh ; \
+            touch .ssh/authorized_keys ; \
+            sed -i.bak -e '/$(awk '{print $NF}' ~/.ssh/id_rsa.pub)/d' .ssh/authorized_keys; \
+            cat tmp.pubkey >> .ssh/authorized_keys ; \
+            rm tmp.pubkey)"
+}
+
+#
+#    Check that host is alive.
+#
+#    \params $1 hostname
+#    \return true if host is alive, false otherwhise.
+#    Example call:
+#        host_alive "mypc"
+#
+host_alive() {
+    host="$1"
+
+    if ping -c 1 "$host" &> /dev/null; then
+        echo "true"
+    else
+        echo "false"
+    fi
+}
+
+#    Check that network interface is valid.
+#
+#    \params $1 interface name
+#    \return true if interface is available, false otherwhise.
+#    Example call:
+#        network_interface_available "eth0"
+#
+network_interface_available() {
+    found=$(grep "$1" /proc/net/dev)
+
+    if  [ -n "$found" ] ; then
+        echo "true"
+    else
+        echo "false"
+    fi
+}
+
+#
+#    Create a temporary file.
+#    Filename will be printed on stdout.
+#
+#    Note: Ubuntu12.04 has an own tempfile function,
+#    but this is not general in all linux distributions.
+tempfile() {
+    filename=$(date +%s | sha256sum | base64 | head -c 32 ; echo)
+    echo "/tmp/$filename"
+}
+
+#
+#    Get a random free tcp port
+#
+#    \return random free port number on stdout
+#    Example:
+#        get_random_port
+#
+get_random_port() {
+    python - <<END
+import socket
+s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+s.bind(("", 0))
+print(s.getsockname()[1])
+s.close()
+END
+}
+
+#
+#    Get owner of a file or directory
+#
+#    \params $@ file/dir
+#    \return on stdout
+#    Example:
+#        get_owner "/home/"
+#
+get_owner() {
+    stat -c %U "$@"
+}
+
+#
+#    Get owner of a file or directory
+#
+#    \params $@ file/dir
+#    \return on stdout
+#    Example:
+#        get_group "/home/"
+#
+get_group() {
+    stat -c %G "$@"
+}
+
+#
+#    Printout settings and check configuration.
+#
+main() {
+    if [[ -z "$_BASHHELPER"  ]]; then
+        _BASHHELPER="included"
+        export _BASHHELPER
+    fi
+}
+
+usage() {
+    echo "helper module"
+}
+
+#
+#    Call main with arguments.
+#
+main "$@"

libs/lib.py

+#!/usr/bin/env python3
+"""
+    lib
+
+    author: Steve Göring
+    contact: stg7@gmx.de
+    2015
+"""
+import re
+import os
+
+
+if __name__ == "__main__":
+    print("lib is not a standalone module")
+    exit(-1)

libs/log.py

+#!/usr/bin/env python3
+"""
+    Logging
+
+    small colored logging functions for python
+
+    author: Steve Göring
+    contact: stg7@gmx.de
+    2015
+"""
+
+
+def colorred(m):
+    return "\033[91m" + m + "\033[0m"
+
+
+def colorblue(m):
+    return "\033[94m" + m + "\033[0m"
+
+
+def colorgreen(m):
+    return "\033[92m" + m + "\033[0m"
+
+
+def colorcyan(m):
+    return "\033[96m" + m + "\033[0m"
+
+
+def lInfo(msg):
+    print(colorgreen("[INFO ] ") + str(msg))
+
+
+def lError(msg):
+    print(colorred("[ERROR] ") + str(msg))
+
+
+def lDbg(msg):
+    print(colorblue("[DEBUG] ") + str(msg))
+
+
+def lWarn(msg):
+    print(colorcyan("[WARN ] ") + str(msg))
+
+
+def lHelp(msg):
+    print(colorblue("[HELP ] ") + str(msg))
+
+if __name__ == "__main__":
+    lError("lib is not a standalone module")
+    exit(-1)

libs/system.py

+#!/usr/bin/env python3
+"""
+    System
+
+    system helper functions
+
+    author: Steve Göring
+    contact: stg7@gmx.de
+    2015
+"""
+import os
+import sys
+import subprocess
+
+
+def shell_call(call):
+    """
+    Run a program via system call and return stdout + stderr.
+    @param call programm and command line parameter list, e.g ["ls", "/"]
+    @return stdout and stderr of programm call
+    """
+    try:
+        output = subprocess.check_output(call, stderr=subprocess.STDOUT, universal_newlines=True)
+    except Exception as e:
+        output = str(e.output)
+    return output
+
+
+def read_file(file_name):
+    """
+    read a text file into a string
+    :file_name file to open
+    :return content as string
+    """
+    f = open(file_name, "r")
+    content = "".join(f.readlines())
+    f.close()
+    return content
+
+
+def get_shebang(file_name):
+    """
+    read shebang of a file
+    :file_name file to open
+    :return shebang
+    """
+    try:
+        f = open(file_name, "r")
+        lines = f.readlines()
+        f.close()
+        if len(lines) == 0:
+            return ""
+        shebang = lines[0]
+        return shebang
+    except Exception as e:
+        return ""
+
+
+def create_dir_if_not_exists(dir):
+    try:
+        os.stat(dir)
+    except:
+        os.mkdir(dir)
+
+
+def get_prog_name():