diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..5602e72 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +* text=auto +*.yml text eol=lf \ No newline at end of file diff --git a/README.md b/README.md index b89573c..41e287b 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,171 @@ -# actions-file-deployer +# FTP/SFTP file deployer -Gitea Action to upload to SFTP / FTP server, optionally via a proxy \ No newline at end of file +This is a fork of https://github.com/milanmk/actions-file-deployer with gitea customisation. + +Fast and customizable deployment with parallel connections and proxy support. Deploy only changed files or do full sync/mirror of repository content. + +This is a composite Gitea Action (Linux runner) for deploying repository content to a remote server. + +## Features + +- Support for FTP and SFTP (SSH) protocols +- Use password or SSH private key for authentication of SFTP connection +- Delta file synchronization for faster deployment of only changed files since last revision +- Mirroring feature to copy entire file and folder structure of repository content +- Optimized for faster file transfers via parallel connections +- Connect to remote server via [SOCKS proxy](https://en.wikipedia.org/wiki/SOCKS) using [SSH tunneling](https://www.ssh.com/academy/ssh/tunneling) to bypass firewall / NAT / IP whitelist / VPC +- Uses [composite action](https://docs.github.com/en/actions/creating-actions/about-actions#types-of-actions) without Docker container for faster deployments and shorter run time +- Pass additional command arguments to SSH and FTP client for custom configurations and settings +- Step runs messages categorized nicely in log groups +- Run additional FTP commands after synchronization + +![Workflow screenshot](./screenshot.png) + +## Usage + +```yml +- name: "Checkout" + uses: actions/checkout@v4 + with: + fetch-depth: 0 +- name: "Deploy" + uses: milanmk/actions-file-deployer@master + with: + remote-protocol: "sftp" + remote-host: "ftp.example.com" + remote-user: "username" + ssh-private-key: ${{ secrets.DEPLOY_PRIVATE_KEY }} + remote-path: "/var/www/example.com" +``` + +Workflow example `.github/workflows/main.yml`. + +```yml +name: Deploy Files + +on: + push: + branches: + - master + # Enables manually triggering of Workflow with file synchronization option + workflow_dispatch: + inputs: + sync: + description: "File synchronization" + required: true + default: "delta" + +jobs: + deploy-master: + name: "master branch" + if: ${{ github.ref == 'refs/heads/master' }} + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: "Checkout" + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: "Deploy" + uses: milanmk/actions-file-deployer@master + with: + remote-protocol: "sftp" + remote-host: "ftp.example.com" + remote-user: "username" + ssh-private-key: ${{ secrets.DEPLOY_PRIVATE_KEY }} + remote-path: "/var/www/example.com" +``` + +## Inputs + +| Name | Required | Default | Description | +|------------------------|----------------------|---------|-----------------------------------------------| +| remote-protocol | yes | sftp | Remote file transfer protocol (ftp, sftp) | +| remote-host | yes | | Remote host | +| remote-port | yes | 22 | Remote port | +| remote-user | yes | | FTP/SSH username | +| remote-password | no | | FTP/SSH password | +| ssh-private-key | no | | SSH private key of user | +| proxy | yes | false | Enable proxy for FTP connection (true, false) | +| proxy-host | yes (if proxy: true) | | Proxy host | +| proxy-port | yes (if proxy: true) | 22 | Proxy port | +| proxy-forwarding-port | yes (if proxy: true) | 1080 | Proxy forwarding port | +| proxy-user | yes (if proxy: true) | | Proxy username | +| proxy-private-key | yes (if proxy: true) | | Proxy SSH private key of user | +| local-path | yes | . | Local path to repository | +| remote-path | yes | . | Remote path on host | +| sync | yes | delta | File synchronization (delta, full) | +| sync-delta-excludes | no | | Files to exclude from delta sync | +| ssh-options | no | | Additional arguments for SSH client | +| ftp-options | no | | Additional arguments for FTP client | +| ftp-mirror-options | no | | Additional arguments for mirroring | +| ftp-post-sync-commands | no | | Additionnal FTP command to run after sync | +| webhook | no | | Send webhook event notifications | +| artifacts | no | false | Upload logs/files to artifacts (true, false) | +| debug | no | false | Publish secrets on the internet (true, false) | + +### Notes + +- Character support for `remote-user` and `remote-password` is limited due to its usage in [.netrc file](https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html) + - It should not contain shell/URL special characters, use [percent encoding](https://en.wikipedia.org/wiki/Percent-encoding) instead +- File synchronization options + - `delta`: Transfer only changed files (upload and delete) since last revision + - Only supported for `push`, `pull_request` and `workflow_dispatch` [events](https://docs.github.com/en/actions/reference/events-that-trigger-workflows) + - Requires `fetch-depth: 0` option in [checkout action](https://gitea.com/actions/checkout) + - It is recommended to initially do a full synchronization and then switch to delta + - `full`: Transfer all files (upload) + - Does not delete files on remote host + - Default glob exclude pattern is `.git*/` +- `sync-delta-excludes` accepts [pathspec](https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddefpathspecapathspec) patterns to exclude files from delta sync. +- For `ftp-options` and `ftp-mirror-options` command arguments please refer to [LFTP manual](https://lftp.yar.ru/lftp-man.html) +- `ftp-post-sync-commands` can be used to run additional LFTP commands after the synchronization. For example, to upload a file watched by a process manager on the server in order to restart a deamon: + ``` + ftp-post-sync-commands: | + !touch watched_file + put watched_file + ``` +- Setting `webhook` to a URL will send start and finish event notifications in JSON format + - start event payload: + ``` + { + "timestamp": "1234567890", + "status": "start", + "repository": "owner/repository", + "workflow": "workflow name", + "job": "deploy", + "run_id": "1234567890", + "ref": "refs/heads/master", + "event_name": "push", + "actor": "username", + "message": "commit message", + "revision": "da39a3ee5e6b4b0d3255bfef95601890afd80709" + } + ``` + - finish event payload: + ``` + { + "timestamp": "1234567890", + "status": "finish", + "repository": "owner/repository", + "workflow": "workflow name", + "job": "deploy", + "run_id": "1234567890", + "ref": "refs/heads/master", + "event_name": "push", + "actor": "username", + "message": "commit message", + "revision": "da39a3ee5e6b4b0d3255bfef95601890afd80709" + } + ``` +- Enabling `artifacts` will upload transfer log and modified files to artifacts + - Modified files are only added for delta file synchronization +- It is strongly recommended to use [Encrypted Secrets](https://docs.github.com/en/actions/reference/encrypted-secrets) to store sensitive data like passwords and private keys +- It is strongly recommended not to use the `debug` option because it will output your encrypted secrets in the Action logs. + +## Planned features +- [ ] Stop the debug option from publishing encrypted secrets +- [x] Add transfer log to artifacts +- [x] Add modified files to artifacts +- [ ] Add steps logging to file +- [ ] Add steps log to artifacts +- [x] Trigger webhook at start and end of step runs diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..3f5af74 --- /dev/null +++ b/action.yml @@ -0,0 +1,397 @@ +name: "FTP/SFTP file deployer" + +description: "Fast and customizable deployment with parallel connections and proxy support. Deploy only changed files or do full sync." + +branding: + icon: "upload" + color: "black" + +inputs: + remote-protocol: + description: "Remote file transfer protocol (ftp, sftp)" + required: true + default: "sftp" + remote-host: + description: "Remote host" + required: true + remote-port: + description: "Remote port" + required: true + default: 22 + remote-user: + description: "FTP/SSH username" + required: true + remote-password: + description: "FTP/SSH password" + required: false + default: "" + ssh-private-key: + description: "SSH private key of user" + required: false + proxy: + description: "Enable proxy for FTP connection (true, false)" + required: true + default: false + proxy-host: + description: "Proxy host" + required: false + proxy-port: + description: "Proxy port" + required: false + default: 22 + proxy-forwarding-port: + description: "Proxy forwarding port" + required: false + default: 1080 + proxy-user: + description: "Proxy username" + required: false + proxy-private-key: + description: "Proxy SSH private key of user" + required: false + local-path: + description: "Local path to repository" + required: true + default: . + remote-path: + description: "Remote path on host" + required: true + default: . + sync: + description: "File synchronization (delta, full)" + required: true + default: "delta" + sync-delta-excludes: + description: "Files to exclude from delta sync" + required: false + ssh-options: + description: "Additional command arguments for SSH client" + required: false + ftp-options: + description: "Additional command arguments for FTP client (lftp)" + required: false + ftp-mirror-options: + description: "Additional command arguments for mirroring (lftp)" + required: false + ftp-post-sync-commands: + description: "Additional ftp commands to run after synchronization (lftp)" + required: false + webhook: + description: "Send webhook event notifications" + required: false + artifacts: + description: "Upload logs to artifacts" + required: false + debug: + description: "Enable debug information (true, false)" + required: false + +runs: + using: "composite" + steps: + - name: "Deploy" + shell: bash + run: | + echo "::group::Initialization" + + function show_hr() { + printf '%.s_' {1..100} && echo "" + } + + function send_webhook() { + local status="$1" + local post_data=$(jq --null-input \ + --arg timestamp `date +%s` \ + --arg status "${status}" \ + --arg repository "${{ gitea.repository }}" \ + --arg workflow "${{ gitea.workflow }}" \ + --arg job "${{ gitea.job }}" \ + --arg run_id "${{ gitea.run_id }}" \ + --arg ref "${{ gitea.ref }}" \ + --arg event_name "${{ gitea.event_name }}" \ + --arg actor "${{ gitea.actor }}" \ + --arg message "${{ gitea.event.head_commit.message }}" \ + --arg revision "${{ gitea.sha }}" \ + '{"timestamp": $timestamp, "status": $status, "repository": $repository, "workflow": $workflow, "job": $job, "run_id": $run_id, "ref": $ref, "event_name": $event_name, "actor": $actor, "message": $message, "revision": $revision}') + + curl --data "${post_data}" --header "Content-Type: application/json" --max-time 30 --show-error --silent --user-agent "Gitea Workflow" "${{inputs.webhook}}" + } + + [ "${{inputs.webhook}}" != "" ] && echo "Webhook notification (start): $(send_webhook "start")" + + echo "Check repository" + + if [ "$(git rev-parse --is-inside-work-tree 2>/dev/null)" != "true" ]; then + echo "::error::Git repository not found. Please ensure you have a checkout step before this step." + exit 1 + fi + + echo "Initialize inputs" + + local_path_unslash=$(echo "${{inputs.local-path}}" | sed 's:/*$::') + local_path_slash="${local_path_unslash}/" + + remote_path_unslash=$(realpath --canonicalize-missing '${{inputs.remote-path}}') + remote_path_slash="${remote_path_unslash}/" + + input_remote_password="${{inputs.remote-password}}" + if [ "${{inputs.remote-password}}" == "" ]; then + input_remote_password="dummypassword" + fi + + input_proxy="${{inputs.proxy}}" + + proxy_cmd="" + if [ "${input_proxy}" == "true" ]; then + proxy_cmd="proxychains" + fi + + input_sync=${{inputs.sync}} + if [ "${{gitea.event_name}}" == "workflow_dispatch" ] && [ -n "${{gitea.event.inputs.sync}}" ]; then + input_sync=${{gitea.event.inputs.sync}} + fi + + echo "Validate inputs" + + if [ "${{inputs.remote-protocol}}" != "sftp" ] && [ "${{inputs.remote-protocol}}" != "ftp" ]; then + echo "::error::Invalid protocol: ${{inputs.remote-protocol}}. Valid protocols are 'ftp' and 'sftp'." + exit 1 + fi + + if [ "${input_sync}" != "delta" ] && [ "${input_sync}" != "full" ]; then + echo "::error::Invalid synchronization: ${input_sync}. Valid types are 'delta' and 'full'." + exit 1 + fi + + echo "::endgroup::" + + + + if [ "${{inputs.debug}}" == "true" ]; then + echo "::group::Debug" + + echo "Context: gitea.event" && cat ${{gitea.event_path}} && show_hr + + echo "Context: env" && echo "${{toJSON(env)}}" && show_hr + + echo "Inputs:" && echo "${{toJSON(inputs)}}" + + echo "::endgroup::" + fi + + + + echo "::group::Install packages" + + apt_install="" + apt_quiet="--quiet --quiet" + if [ "${{inputs.debug}}" == "true" ]; then + apt_quiet="" + fi + + sudo apt-get ${apt_quiet} update && sudo apt-get ${apt_quiet} --no-install-recommends --yes install lftp ${proxy_cmd} + + echo "::endgroup::" + + + + echo "::group::Configurations" + + config_ssh=~/.ssh/config + mkdir ~/.ssh && echo -e "ExitOnForwardFailure=yes\nStrictHostKeyChecking=no" > ${config_ssh} && chmod 600 ${config_ssh} && echo "File created: ${config_ssh}" + [ "${{inputs.debug}}" == "true" ] && cat ${config_ssh} + show_hr + + netrc=~/.netrc + echo "machine ${{inputs.remote-host}} login ${{inputs.remote-user}} password ${input_remote_password}" > ${netrc} && chmod 600 ${netrc} && echo "File created: ${netrc}" + [ "${{inputs.debug}}" == "true" ] && cat ${netrc} + show_hr + + if [ "${{inputs.remote-protocol}}" == "sftp" ] && [ "${{inputs.ssh-private-key}}" != "" ]; then + key_ssh=~/ssh_private_key + echo "${{inputs.ssh-private-key}}" > ${key_ssh} && chmod 600 ${key_ssh} && echo "File created: ${key_ssh}" && show_hr + fi + + if [ "${input_proxy}" == "true" ]; then + if [ "${{inputs.proxy-private-key}}" != "" ]; then + key_proxy=~/proxy_private_key + echo "${{inputs.proxy-private-key}}" > ${key_proxy} && chmod 600 ${key_proxy} && echo "File created: ${key_proxy}" && show_hr + + config_proxychains=~/.proxychains/proxychains.conf + mkdir ~/.proxychains && echo "strict_chain + quiet_mode + tcp_read_time_out 15000 + tcp_connect_time_out 10000 + [ProxyList] + socks5 127.0.0.1 ${{inputs.proxy-forwarding-port}}" > ${config_proxychains} && echo "File created: ${config_proxychains}" + [ "${{inputs.debug}}" == "true" ] && cat ${config_proxychains} + show_hr + else + input_proxy="false" + echo "::warning::Invalid input 'proxy-private-key'. Skipping proxy connection." + fi + fi + + echo "debug $([ "${{inputs.debug}}" == "true" ] && echo "9" || echo "false") + set cmd:trace $([ "${{inputs.debug}}" == "true" ] && echo "true" || echo "false") + set ftp:ssl-protect-data true + set ftp:sync-mode false + set log:enabled/xfer true + set log:file/xfer ~/transfer_log.txt + set log:show-time/xfer false + set mirror:overwrite true + set mirror:parallel-transfer-count 3 + set mirror:set-permissions false + set net:max-retries 1 + set net:persist-retries 0 + set net:timeout 10 + set sftp:auto-confirm true + set ssl:check-hostname false + set ssl:verify-certificate false + set xfer:parallel 3 + ${{inputs.ftp-options}}" > ~/.lftprc + if [ "${{inputs.remote-protocol}}" == "sftp" ] && [ "${{inputs.ssh-private-key}}" != "" ]; then + echo "set sftp:connect-program /usr/bin/ssh -a -x -i ~/ssh_private_key ${{inputs.ssh-options}}" >> ~/.lftprc + else + echo "set sftp:connect-program /usr/bin/ssh -a -x ${{inputs.ssh-options}}" >> ~/.lftprc + fi + echo "open ${{inputs.remote-protocol}}://${{inputs.remote-user}}@${{inputs.remote-host}}:${{inputs.remote-port}}" >> ~/.lftprc + echo "File created: ~/.lftprc" + [ "${{inputs.debug}}" == "true" ] && cat ~/.lftprc + + echo "::endgroup::" + + + + if [ "${input_proxy}" == "true" ]; then + echo "::group::Setup proxy" + + if [ "${{inputs.proxy-user}}" != "" ] && [ "${{inputs.proxy-host}}" != "" ]; then + if ssh -A -D ${{inputs.proxy-forwarding-port}} -f -N -p ${{inputs.proxy-port}} -i ~/proxy_private_key ${{inputs.proxy-user}}@${{inputs.proxy-host}}; then + echo "Proxy connected" && show_hr && echo "Proxy IP address: $(${proxy_cmd} curl --max-time 10 --show-error --silent "http://checkip.amazonaws.com/")" + else + echo "::error::Proxy connection failed." + exit 1 + fi + else + input_proxy="false" + echo "::warning::Invalid input 'proxy-user', 'proxy-host'. Skipping proxy connection." + fi + + echo "::endgroup::" + fi + + + + echo "::group::Prepare files" + + echo "Event: ${{gitea.event_name}} + Revision: https://gitea.com/${{gitea.repository}}/commit/${{gitea.sha}} + Committer: ${{gitea.actor}} + Message: ${{gitea.event.head_commit.message}}" && show_hr + + echo "${{gitea.sha}}" > "${local_path_slash}.deploy-revision" && echo "File created: ${local_path_slash}.deploy-revision" && cat "${local_path_slash}.deploy-revision" && show_hr + + if [ "${input_sync}" == "delta" ]; then + touch ~/files_to_upload ~/files_to_delete + git_depth=$(git rev-list --count --all) + git_previous_commit="" + + if [ "${git_depth}" -gt 1 ]; then + if [ "${{gitea.event_name}}" == "push" ]; then + git_previous_commit=${{gitea.event.before}} + elif [ "${{gitea.event_name}}" == "pull_request" ]; then + git_previous_commit=${{gitea.event.pull_request.base.sha}} + elif [ "${{gitea.event_name}}" == "workflow_dispatch" ]; then + git_previous_commit=$(git rev-parse ${{gitea.sha}}^) + else + echo "::error::Event not supported for delta synchronization: ${{gitea.event_name}}. Supported events are 'push', 'pull_request' and 'workflow_dispatch'." + exit 1 + fi + else + echo "::error::Commit history not found for delta synchronization. Please ensure you have 'fetch-depth: 0' option in checkout action. Please ignore if this is an initial commit or newly created branch." + exit 1 + fi + domain="code.klank.school" + echo "Previous Revision: https://${domain}/${{gitea.repository}}/commit/${git_previous_commit}" && show_hr + + # ${proxy_cmd} lftp -c "set log:enabled/xfer false; get -O ~ \"${remote_path_slash}.deploy-revision\"; exit 0" + # echo -n "Remote Revision: " && [ -f ~/.deploy-revision ] && cat ~/.deploy-revision || echo "" + # show_hr + + if git cat-file -t ${git_previous_commit} &>/dev/null; then + git diff --diff-filter=ACMRT --name-only ${git_previous_commit}..${{gitea.sha}} -- ${local_path_unslash} ':!/.git*' ${{inputs.sync-delta-excludes}} > ~/files_to_upload + git diff-tree --diff-filter=D --name-only -t ${git_previous_commit}..${{gitea.sha}} -- ${local_path_unslash} ':!/.git*' ${{inputs.sync-delta-excludes}} > ~/files_to_delete + + sed --in-place --regexp-extended "s#(.*)#realpath --canonicalize-missing --relative-to=$local_path_unslash \1#e" ~/files_to_upload + sed --in-place --regexp-extended "s#(.*)#realpath --canonicalize-missing --relative-to=$local_path_unslash \1#e" ~/files_to_delete + + echo "File created: ~/files_to_upload" && cat ~/files_to_upload && show_hr + echo "File created: ~/files_to_delete" && cat ~/files_to_delete && show_hr + + if [ "${{inputs.artifacts}}" == "true" ]; then + echo "Copy transfer artifacts" && mkdir ~/transfer_files && rsync --verbose --files-from=$HOME/files_to_upload ${local_path_slash} ~/transfer_files/ + fi + else + echo "::warning::Invalid base commit for delta synchronization: ${git_previous_commit}. Please ignore if this is an initial commit or newly created branch." + fi + fi + + echo "::endgroup::" + + + + echo "::group::Transfer files" + + echo "Protocol: ${{inputs.remote-protocol}} + Synchronization: ${input_sync} + Local path: ${local_path_unslash} + Remote path: ${remote_path_unslash}" + + [ "${input_sync}" == "delta" ] && echo -e "Upload files: $(wc --lines < ~/files_to_upload)\nDelete files: $(wc --lines < ~/files_to_delete)" + + show_hr + + touch "${local_path_slash}.deploy-running" + + if [ "${input_sync}" == "full" ]; then + ${proxy_cmd} lftp -c "put -O \"${remote_path_unslash}\" \"${local_path_slash}.deploy-running\"; + mirror --exclude-glob=.git*/ --max-errors=10 --reverse ${{inputs.ftp-mirror-options}} ${local_path_unslash} ${remote_path_unslash}; + rm -f \"${remote_path_slash}.deploy-running\"; + ${{inputs.ftp-post-sync-commands}}" + else + ${proxy_cmd} lftp -c "lcd \"${local_path_unslash}\"; + put -O \"${remote_path_unslash}\" \"${local_path_slash}.deploy-running\"; + mput -d -O \"${remote_path_unslash}\" .deploy-revision $(awk '{ printf "\"%s\" ", $0 }' ~/files_to_upload); + rm -f \"${remote_path_slash}.deploy-check\" $(awk -v REMOTEPATH=\"${remote_path_slash}\" '{ printf "\"%s%s\" ", REMOTEPATH, $0 }' ~/files_to_delete); + rm -f \"${remote_path_slash}.deploy-running\"; + ${{inputs.ftp-post-sync-commands}}" + fi + + [ -f ~/transfer_log.txt ] && cat ~/transfer_log.txt + + echo "::endgroup::" + + + + echo "::group::Cleanup" + + [ "${input_proxy}" == "true" ] && sudo pkill ssh + + rm --force --verbose ~/.netrc ~/proxy_private_key ~/ssh_private_key + + [ "${{inputs.artifacts}}" != "true" ] && rm --force --verbose ~/transfer_log.txt + + [ "${{inputs.webhook}}" != "" ] && echo "Webhook notification (finish): $(send_webhook "finish")" + + echo "::endgroup::" + + exit 0 + - name: "Upload artifacts" + uses: actions/upload-artifact@v4 + with: + name: "transfer_artifacts" + path: | + ~/transfer_log.txt + ~/transfer_files + if-no-files-found: ignore diff --git a/screenshot.png b/screenshot.png new file mode 100644 index 0000000..9934484 Binary files /dev/null and b/screenshot.png differ