diff --git a/.gitignore b/.gitignore index 043359d..6f758b8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ .idea -test.sh +test.* .env \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 9aed1f6..81a5e1b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,27 @@ -FROM alpine:latest +FROM python:3.8.3-slim-buster LABEL "maintainer"="Scott Ng " -LABEL "repository"="https://github.com/cross-the-world/scp-pipeline" -LABEL "version"="1.0.0" +LABEL "repository"="https://github.com/cross-the-world/ssh-pipeline" +LABEL "version"="latest" LABEL "com.github.actions.name"="scp-pipeline" LABEL "com.github.actions.description"="Pipeline: scp" LABEL "com.github.actions.icon"="copy" LABEL "com.github.actions.color"="gray-dark" -RUN apk update && \ - apk add ca-certificates && \ - apk add --no-cache openssh-client openssl openssh sshpass && \ - apk add --no-cache --upgrade bash openssh sshpass && \ - rm -rf /var/cache/apk/* +RUN apt-get update -y && \ + apt-get install -y ca-certificates openssh-client openssl sshpass -COPY entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh +COPY requirements.txt /requirements.txt +RUN pip3 install -r /requirements.txt -ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file +RUN mkdir -p /opt/tools +WORKDIR /opt/tools + +COPY entrypoint.sh /opt/tools/entrypoint.sh +RUN chmod +x /opt/tools/entrypoint.sh + +COPY app.py /opt/tools/app.py +RUN chmod +x /opt/tools/app.py + +ENTRYPOINT ["./entrypoint.sh"] \ No newline at end of file diff --git a/app.py b/app.py new file mode 100644 index 0000000..7da3dfc --- /dev/null +++ b/app.py @@ -0,0 +1,93 @@ +from os import environ, path +from glob import glob + +import paramiko +import scp +import sys +import math +import re + + +envs = environ +INPUT_HOST = envs.get("INPUT_HOST") +INPUT_PORT = int(envs.get("INPUT_PORT", "22")) +INPUT_USER = envs.get("INPUT_USER") +INPUT_PASS = envs.get("INPUT_PASS") +INPUT_KEY = envs.get("INPUT_KEY") +INPUT_CONNECT_TIMEOUT = envs.get("INPUT_CONNECT_TIMEOUT", "30s") +INPUT_SCP = envs.get("INPUT_SCP") +INPUT_LOCAL = envs.get("INPUT_LOCAL") +INPUT_REMOTE = envs.get("INPUT_REMOTE") + + +seconds_per_unit = {"s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800, "M": 86400*30} +pattern_seconds_per_unit = re.compile(r'^(' + "|".join(['\\d+'+k for k in seconds_per_unit.keys()]) + ')$') + + +def convert_to_seconds(s): + if s is None: + return 30 + if isinstance(s, str): + return int(s[:-1]) * seconds_per_unit[s[-1]] if pattern_seconds_per_unit.search(s) else 30 + if (isinstance(s, int) or isinstance(s, float)) and not math.isnan(s): + return round(s) + return 30 + + +def connect(): + ssh = paramiko.SSHClient() + p_key = paramiko.RSAKey.from_private_key(INPUT_KEY) if INPUT_KEY else None + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(INPUT_HOST, port=INPUT_PORT, username=INPUT_USER, + pkey=p_key, password=INPUT_PASS, + timeout=convert_to_seconds(INPUT_CONNECT_TIMEOUT)) + return ssh + + +# Define progress callback that prints the current percentage completed for the file +def progress(filename, size, sent): + sys.stdout.write(f"{filename} copying: {float(sent)/float(size)*100:.2f}") + + +def scp_process(): + if (INPUT_KEY is None and INPUT_PASS is None) or (not INPUT_SCP and not (INPUT_LOCAL and INPUT_REMOTE)): + print("SCP invalid (Script/Key/Passwd)") + return + + print("+++++++++++++++++++Pipeline: RUNNING SCP+++++++++++++++++++") + + copy_list = [] + if INPUT_LOCAL and INPUT_REMOTE: + copy_list.append({"l": INPUT_LOCAL, "r": INPUT_REMOTE}) + for c in INPUT_SCP.splitlines(): + if not c: + continue + l2r = c.split("=>") + if len(l2r) == 2: + local = l2r[0].strip() + remote = l2r[1].strip() + if local and remote: + copy_list.append({"l": local, "r": remote}) + continue + print(f"SCP ignored {c.strip()}") + print(copy_list) + + if len(copy_list) <= 0: + print("SCP no copy list found") + return + + ssh = connect() + with scp.SCPClient(ssh.get_transport(), progress=progress, sanitize=lambda x: x) as conn: + for l2r in copy_list: + remote = l2r.get('r') + ssh.exec_command(f"mkdir -p {remote} || true") + files = [f for f in glob(l2r.get('l'))] + conn.put(files, remote_path=remote, recursive=True) + for f in files: + print(f"{f} -> {remote}") + + +if __name__ == '__main__': + scp_process() + + diff --git a/entrypoint.sh b/entrypoint.sh index d07f435..90a86f4 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,100 +1,7 @@ #!/bin/bash -set -e - -createKeyFile() { - local SSH_PATH="$HOME/.ssh" - - mkdir -p "$SSH_PATH" - touch "$SSH_PATH/known_hosts" - - echo "$INPUT_KEY" > "$SSH_PATH/id_rsa" - - chmod 700 "$SSH_PATH" - chmod 600 "$SSH_PATH/known_hosts" - chmod 600 "$SSH_PATH/id_rsa" - - eval $(ssh-agent) - ssh-add "$SSH_PATH/id_rsa" - - ssh-keyscan -t rsa "$INPUT_HOST" >> "$SSH_PATH/known_hosts" -} - -check_remote_dir() { - local USEPASS=$1 - local REMOTE=$2 - local CMD="ssh" - if $USEPASS; then - CMD="sshpass -p $INPUT_PASS ssh" - fi - echo "Checking remote directory: '$REMOTE'" - if $CMD -o StrictHostKeyChecking=no -o ConnectTimeout=${INPUT_CONNECT_TIMEOUT:-30s} -p "${INPUT_PORT:-22}" "$INPUT_USER"@"$INPUT_HOST" "[ ! -d $REMOTE ]"; then - echo "Creating: '$REMOTE' on '$INPUT_USER'@'$INPUT_HOST'" - $CMD -o StrictHostKeyChecking=no -o ConnectTimeout=${INPUT_CONNECT_TIMEOUT:-30s} -p "${INPUT_PORT:-22}" "$INPUT_USER"@"$INPUT_HOST" "mkdir -p $REMOTE" - else - echo "'$REMOTE' exists [OK]" - fi -} - -executeSCP() { - local USEPASS=$1 - local LINES=$2 - local COMMAND= - - local CMD="scp" - if $USEPASS; then - CMD="sshpass -p $INPUT_PASS scp" - fi - - while IFS= read -r LINE; do - delimiter="=>" - LINE=$(echo $LINE) - if [[ -z "${LINE}" ]]; then - continue - fi - s=$LINE$delimiter - arr=() - while [[ $s ]]; do - arr+=( "${s%%"$delimiter"*}" ); - s=${s#*"$delimiter"}; - done; - LOCAL=$(eval 'echo "${arr[0]}"') - LOCAL=$(eval echo "$LOCAL") - REMOTE=$(eval 'echo "${arr[1]}"') - REMOTE=$(eval echo "$REMOTE") - - if [[ -z "${LOCAL}" ]] || [[ -z "${REMOTE}" ]]; then - echo "LOCAL/REMOTE can not be parsed $LINE" - else - check_remote_dir $USEPASS $REMOTE - echo "Copying $LOCAL ---> $REMOTE" - $CMD -r -o StrictHostKeyChecking=no -o ConnectTimeout=${INPUT_CONNECT_TIMEOUT:-30s} -P "${INPUT_PORT:-22}" $LOCAL "$INPUT_USER"@"$INPUT_HOST":$REMOTE > /dev/stdout - fi - done <<< "$LINES" -} - - -###################################################################################### - echo "+++++++++++++++++++STARTING PIPELINES+++++++++++++++++++" -USEPASS=true -if [[ -z "${INPUT_KEY}" ]]; then - echo "+++++++++++++++++++Use password+++++++++++++++++++" -else - echo "+++++++++++++++++++Create Key File+++++++++++++++++++" - USEPASS=false - createKeyFile || false -fi - -if ! [[ -z "${INPUT_LOCAL}" ]] && ! [[ -z "${INPUT_REMOTE}" ]]; then - echo "+++++++++++++++++++Pipeline: LOCAL -> REMOTE+++++++++++++++++++" - executeSCP "$USEPASS" "$INPUT_LOCAL => $INPUT_REMOTE" || false -fi - -if ! [[ -z "${INPUT_SCP}" ]]; then - echo "+++++++++++++++++++Pipeline: RUNNING SCP+++++++++++++++++++" - executeSCP "$USEPASS" "$INPUT_SCP" || false -fi +python3 /opt/tools/app.py echo "+++++++++++++++++++END PIPELINES+++++++++++++++++++" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a46da83 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +paramiko +scp \ No newline at end of file