mirror of
https://github.com/cross-the-world/scp-pipeline.git
synced 2025-09-10 11:39:52 +08:00
action using python
This commit is contained in:
Nguyen Huu Thuong
2
.gitignore
vendored
2
.gitignore
vendored
@@ -1,4 +1,4 @@
|
||||
.idea
|
||||
|
||||
test.sh
|
||||
test.*
|
||||
.env
|
||||
28
Dockerfile
28
Dockerfile
@@ -1,21 +1,27 @@
|
||||
FROM alpine:latest
|
||||
FROM python:3.8.3-slim-buster
|
||||
|
||||
LABEL "maintainer"="Scott Ng <thuongnht@gmail.com>"
|
||||
LABEL "repository"="https://github.com/cross-the-world/scp-pipeline"
|
||||
LABEL "version"="1.0.0"
|
||||
LABEL "repository"="https://github.com/cross-the-world/ssh-pipeline"
|
||||
LABEL "version"="latest"
|
||||
|
||||
LABEL "com.github.actions.name"="scp-pipeline"
|
||||
LABEL "com.github.actions.description"="Pipeline: scp"
|
||||
LABEL "com.github.actions.icon"="copy"
|
||||
LABEL "com.github.actions.color"="gray-dark"
|
||||
|
||||
RUN apk update && \
|
||||
apk add ca-certificates && \
|
||||
apk add --no-cache openssh-client openssl openssh sshpass && \
|
||||
apk add --no-cache --upgrade bash openssh sshpass && \
|
||||
rm -rf /var/cache/apk/*
|
||||
RUN apt-get update -y && \
|
||||
apt-get install -y ca-certificates openssh-client openssl sshpass
|
||||
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
COPY requirements.txt /requirements.txt
|
||||
RUN pip3 install -r /requirements.txt
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
RUN mkdir -p /opt/tools
|
||||
WORKDIR /opt/tools
|
||||
|
||||
COPY entrypoint.sh /opt/tools/entrypoint.sh
|
||||
RUN chmod +x /opt/tools/entrypoint.sh
|
||||
|
||||
COPY app.py /opt/tools/app.py
|
||||
RUN chmod +x /opt/tools/app.py
|
||||
|
||||
ENTRYPOINT ["./entrypoint.sh"]
|
||||
93
app.py
Normal file
93
app.py
Normal file
@@ -0,0 +1,93 @@
|
||||
from os import environ, path
|
||||
from glob import glob
|
||||
|
||||
import paramiko
|
||||
import scp
|
||||
import sys
|
||||
import math
|
||||
import re
|
||||
|
||||
|
||||
envs = environ
|
||||
INPUT_HOST = envs.get("INPUT_HOST")
|
||||
INPUT_PORT = int(envs.get("INPUT_PORT", "22"))
|
||||
INPUT_USER = envs.get("INPUT_USER")
|
||||
INPUT_PASS = envs.get("INPUT_PASS")
|
||||
INPUT_KEY = envs.get("INPUT_KEY")
|
||||
INPUT_CONNECT_TIMEOUT = envs.get("INPUT_CONNECT_TIMEOUT", "30s")
|
||||
INPUT_SCP = envs.get("INPUT_SCP")
|
||||
INPUT_LOCAL = envs.get("INPUT_LOCAL")
|
||||
INPUT_REMOTE = envs.get("INPUT_REMOTE")
|
||||
|
||||
|
||||
seconds_per_unit = {"s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800, "M": 86400*30}
|
||||
pattern_seconds_per_unit = re.compile(r'^(' + "|".join(['\\d+'+k for k in seconds_per_unit.keys()]) + ')$')
|
||||
|
||||
|
||||
def convert_to_seconds(s):
|
||||
if s is None:
|
||||
return 30
|
||||
if isinstance(s, str):
|
||||
return int(s[:-1]) * seconds_per_unit[s[-1]] if pattern_seconds_per_unit.search(s) else 30
|
||||
if (isinstance(s, int) or isinstance(s, float)) and not math.isnan(s):
|
||||
return round(s)
|
||||
return 30
|
||||
|
||||
|
||||
def connect():
|
||||
ssh = paramiko.SSHClient()
|
||||
p_key = paramiko.RSAKey.from_private_key(INPUT_KEY) if INPUT_KEY else None
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect(INPUT_HOST, port=INPUT_PORT, username=INPUT_USER,
|
||||
pkey=p_key, password=INPUT_PASS,
|
||||
timeout=convert_to_seconds(INPUT_CONNECT_TIMEOUT))
|
||||
return ssh
|
||||
|
||||
|
||||
# Define progress callback that prints the current percentage completed for the file
|
||||
def progress(filename, size, sent):
|
||||
sys.stdout.write(f"{filename} copying: {float(sent)/float(size)*100:.2f}")
|
||||
|
||||
|
||||
def scp_process():
|
||||
if (INPUT_KEY is None and INPUT_PASS is None) or (not INPUT_SCP and not (INPUT_LOCAL and INPUT_REMOTE)):
|
||||
print("SCP invalid (Script/Key/Passwd)")
|
||||
return
|
||||
|
||||
print("+++++++++++++++++++Pipeline: RUNNING SCP+++++++++++++++++++")
|
||||
|
||||
copy_list = []
|
||||
if INPUT_LOCAL and INPUT_REMOTE:
|
||||
copy_list.append({"l": INPUT_LOCAL, "r": INPUT_REMOTE})
|
||||
for c in INPUT_SCP.splitlines():
|
||||
if not c:
|
||||
continue
|
||||
l2r = c.split("=>")
|
||||
if len(l2r) == 2:
|
||||
local = l2r[0].strip()
|
||||
remote = l2r[1].strip()
|
||||
if local and remote:
|
||||
copy_list.append({"l": local, "r": remote})
|
||||
continue
|
||||
print(f"SCP ignored {c.strip()}")
|
||||
print(copy_list)
|
||||
|
||||
if len(copy_list) <= 0:
|
||||
print("SCP no copy list found")
|
||||
return
|
||||
|
||||
ssh = connect()
|
||||
with scp.SCPClient(ssh.get_transport(), progress=progress, sanitize=lambda x: x) as conn:
|
||||
for l2r in copy_list:
|
||||
remote = l2r.get('r')
|
||||
ssh.exec_command(f"mkdir -p {remote} || true")
|
||||
files = [f for f in glob(l2r.get('l'))]
|
||||
conn.put(files, remote_path=remote, recursive=True)
|
||||
for f in files:
|
||||
print(f"{f} -> {remote}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
scp_process()
|
||||
|
||||
|
||||
@@ -1,100 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
createKeyFile() {
|
||||
local SSH_PATH="$HOME/.ssh"
|
||||
|
||||
mkdir -p "$SSH_PATH"
|
||||
touch "$SSH_PATH/known_hosts"
|
||||
|
||||
echo "$INPUT_KEY" > "$SSH_PATH/id_rsa"
|
||||
|
||||
chmod 700 "$SSH_PATH"
|
||||
chmod 600 "$SSH_PATH/known_hosts"
|
||||
chmod 600 "$SSH_PATH/id_rsa"
|
||||
|
||||
eval $(ssh-agent)
|
||||
ssh-add "$SSH_PATH/id_rsa"
|
||||
|
||||
ssh-keyscan -t rsa "$INPUT_HOST" >> "$SSH_PATH/known_hosts"
|
||||
}
|
||||
|
||||
check_remote_dir() {
|
||||
local USEPASS=$1
|
||||
local REMOTE=$2
|
||||
local CMD="ssh"
|
||||
if $USEPASS; then
|
||||
CMD="sshpass -p $INPUT_PASS ssh"
|
||||
fi
|
||||
echo "Checking remote directory: '$REMOTE'"
|
||||
if $CMD -o StrictHostKeyChecking=no -o ConnectTimeout=${INPUT_CONNECT_TIMEOUT:-30s} -p "${INPUT_PORT:-22}" "$INPUT_USER"@"$INPUT_HOST" "[ ! -d $REMOTE ]"; then
|
||||
echo "Creating: '$REMOTE' on '$INPUT_USER'@'$INPUT_HOST'"
|
||||
$CMD -o StrictHostKeyChecking=no -o ConnectTimeout=${INPUT_CONNECT_TIMEOUT:-30s} -p "${INPUT_PORT:-22}" "$INPUT_USER"@"$INPUT_HOST" "mkdir -p $REMOTE"
|
||||
else
|
||||
echo "'$REMOTE' exists [OK]"
|
||||
fi
|
||||
}
|
||||
|
||||
executeSCP() {
|
||||
local USEPASS=$1
|
||||
local LINES=$2
|
||||
local COMMAND=
|
||||
|
||||
local CMD="scp"
|
||||
if $USEPASS; then
|
||||
CMD="sshpass -p $INPUT_PASS scp"
|
||||
fi
|
||||
|
||||
while IFS= read -r LINE; do
|
||||
delimiter="=>"
|
||||
LINE=$(echo $LINE)
|
||||
if [[ -z "${LINE}" ]]; then
|
||||
continue
|
||||
fi
|
||||
s=$LINE$delimiter
|
||||
arr=()
|
||||
while [[ $s ]]; do
|
||||
arr+=( "${s%%"$delimiter"*}" );
|
||||
s=${s#*"$delimiter"};
|
||||
done;
|
||||
LOCAL=$(eval 'echo "${arr[0]}"')
|
||||
LOCAL=$(eval echo "$LOCAL")
|
||||
REMOTE=$(eval 'echo "${arr[1]}"')
|
||||
REMOTE=$(eval echo "$REMOTE")
|
||||
|
||||
if [[ -z "${LOCAL}" ]] || [[ -z "${REMOTE}" ]]; then
|
||||
echo "LOCAL/REMOTE can not be parsed $LINE"
|
||||
else
|
||||
check_remote_dir $USEPASS $REMOTE
|
||||
echo "Copying $LOCAL ---> $REMOTE"
|
||||
$CMD -r -o StrictHostKeyChecking=no -o ConnectTimeout=${INPUT_CONNECT_TIMEOUT:-30s} -P "${INPUT_PORT:-22}" $LOCAL "$INPUT_USER"@"$INPUT_HOST":$REMOTE > /dev/stdout
|
||||
fi
|
||||
done <<< "$LINES"
|
||||
}
|
||||
|
||||
|
||||
######################################################################################
|
||||
|
||||
echo "+++++++++++++++++++STARTING PIPELINES+++++++++++++++++++"
|
||||
|
||||
USEPASS=true
|
||||
if [[ -z "${INPUT_KEY}" ]]; then
|
||||
echo "+++++++++++++++++++Use password+++++++++++++++++++"
|
||||
else
|
||||
echo "+++++++++++++++++++Create Key File+++++++++++++++++++"
|
||||
USEPASS=false
|
||||
createKeyFile || false
|
||||
fi
|
||||
|
||||
if ! [[ -z "${INPUT_LOCAL}" ]] && ! [[ -z "${INPUT_REMOTE}" ]]; then
|
||||
echo "+++++++++++++++++++Pipeline: LOCAL -> REMOTE+++++++++++++++++++"
|
||||
executeSCP "$USEPASS" "$INPUT_LOCAL => $INPUT_REMOTE" || false
|
||||
fi
|
||||
|
||||
if ! [[ -z "${INPUT_SCP}" ]]; then
|
||||
echo "+++++++++++++++++++Pipeline: RUNNING SCP+++++++++++++++++++"
|
||||
executeSCP "$USEPASS" "$INPUT_SCP" || false
|
||||
fi
|
||||
python3 /opt/tools/app.py
|
||||
|
||||
echo "+++++++++++++++++++END PIPELINES+++++++++++++++++++"
|
||||
|
||||
2
requirements.txt
Normal file
2
requirements.txt
Normal file
@@ -0,0 +1,2 @@
|
||||
paramiko
|
||||
scp
|
||||
Reference in New Issue
Block a user