diff --git a/bin/tus b/bin/tus index ff382c6..b5099d1 100755 --- a/bin/tus +++ b/bin/tus @@ -1,29 +1,29 @@ #!/usr/bin/env bash set -e -: ${TUSD:=""} -: ${TUS_HEADERS:=''} +: "${TUSD:=}" +: "${TUS_HEADERS:=}" export TUS_CHUNK_SIZE=${TUS_CHUNK_SIZE:-1048576} export TUS_PARALLEL=${TUS_PARALLEL:-1} # ============================================================================== # debugging & usage & tus-helper -function error { local message="${1}"; local env="${2}"; - [ -n "${2}" ] && message="Missing ${1}. Please execute:\n" +function error { local message="$1"; local env="$2"; + [ -n "$2" ] && message="Missing $1. Please execute:\n" - echo -e "\033[1;31m➤\033[0m ${message}" - [ -n "${env}" ] \ - && echo -e "\033[1;34mexport\033[0m ${env}=\"\033[1;31m<<${1}>>\033[0m\"" + echo -e "\033[1;31m➤\033[0m $message" + [ -n "$env" ] \ + && echo -e "\033[1;34mexport\033[0m $env=\"\033[1;31m<<$1>>\033[0m\"" exit 1 } -function warning { local message="${1}" - echo -e "\033[0;33m➤\033[0m ${message}" +function warning { local message="$1" + echo -e "\033[0;33m➤\033[0m $message" } -function usage { local message="${1}"; - [ -n "${message}" ] \ - && echo -e "\033[0;31m➤\033[0m \033[1m${message}\033[0m" +function usage { local message="$1"; + [ -n "$message" ] \ + && echo -e "\033[0;31m➤\033[0m \033[1m$message\033[0m" cat <<-EOTXT Usage: @@ -39,27 +39,39 @@ Options: -H HEADER Set additional header. -r Reuploads given file from the beginning. -h Shows usage. + -D Debug. EOTXT echo -e "\033[2m➤ https://tus.io/protocols/resumable-upload.html\033[0m" echo -e "\033[2m➤ https://github.com/fentas/tus-shell-client\033[0m" - [ -z "${message}" ] && exit 0 || exit 1 + [ -z "$message" ] && exit 0 || exit 1 } # ============================================================================== # helper -function ceil { local dividend="${1}"; local divisor="${2}" - [ $(( dividend % divisor )) -gt 0 ] \ - && echo $(( ( ( $dividend - ( $dividend % $divisor ) ) / $divisor ) + 1 )) \ - || echo $(( $dividend / $divisor )) + +# https://stackoverflow.com/a/34545932/10440128 +function ceildiv() { + local num=${1:-0} + local div=${2:-1} + if ! ((div)); then return 1; fi + if ((num >= 0)); then + echo $(( (num + div - 1) / div )) + else + echo $(( -(-num + div - 1) / div )) + fi } -function expand { local prefix="${1}"; shift; for entry in "${@}"; do - [ ! -z "$entry" ] && echo -e "$prefix \"$entry\""; -done; } +function set_header_args() { + # note: nonlocal variable: header_args + local header + for header in "$@"; do + header_args+=("-H" "$header") + done +} -function strip-domain { for entry in "${@}"; do - [ ! -z "$entry" ] && echo -n "${entry} " | sed -E 's|^https?://[^/]+(.*)|\1|'; +function strip-domain { for entry in "$@"; do + [ -n "$entry" ] && echo -n "$entry " | sed -E 's|^https?://[^/]+(.*)|\1|'; done; } function join { local IFS="$1"; shift @@ -68,149 +80,154 @@ function join { local IFS="$1"; shift # ============================================================================== # tus -tus_location=$(mktemp -u) -mkfifo $tus_location -function tus_options { +function get_tus_options { # The Client SHOULD NOT include the Tus-Resumable header in the request and the Server MUST ignore the header. - eval "local tus_headers=($(expand '-H' "${TUS_HEADERS[@]}"))" - curl -s \ - "${tus_headers[@]}" \ + local header_args=() + set_header_args "${TUS_HEADERS[@]}" + local response + local rc + response="$(curl -s \ + --fail-with-body \ + "${header_args[@]}" \ --head \ --request OPTIONS \ - ${TUSD} | grep 'Tus-' | tr -d '\015' + "$TUSD" + )" + rc=$? + if [ $rc != 0 ]; then + echo "$response" | tr -d '\r' + return $rc + fi + echo "$response" | grep -i '^Tus-' | tr -d '\r' } function tus_supports { - echo "${TUS_OPTIONS[@]}" | grep -qE "^Tus-Extension:.*[, ]${1}[,$]" \ - || error "${2}.\n\033[2m↳ See tusd options: \033[1;34m${0} -o\033[0m" + echo "${TUS_OPTIONS[@]}" | grep -qE "^Tus-Extension:.*[, ]$1[,$]" \ + || error "$2.\n\033[2m↳ See tusd options: \033[1;34m$0 -o\033[0m" } function tus_create { - local file="${1}"; shift - local size="${1}"; shift + local file="$1"; shift + local size="$1"; shift - eval "local tus_headers=($(expand '-H' "${TUS_HEADERS[@]}"))" - for header in "${@}"; do - tus_headers+=(-H "${header}") + local header_args=() + set_header_args "${TUS_HEADERS[@]}" + for header in "$@"; do + header_args+=(-H "$header") done - # Apparently 'Location: ..' is terminated by CRLF. grep and awk faithfully - # preserve the line ending, and the shell's $() substitution strips off the - # final LF leaving you with a string that just ends with a CR. - # - # When the CR is printed, the cursor moves to the beginning of the line and - # whatever gets printed next overwrites what was there. - # ... | tr -d '\015' + # Most HTTP server use '\r\n' line endings, so here we remove the '\r' curl -s \ - "${tus_headers[@]}" \ + --fail-with-body \ + "${header_args[@]}" \ --head \ --request POST \ --header "Tus-Resumable: 1.0.0" \ --header "Content-Length: 0" \ - --header "Upload-Length: ${size}" \ - --header "Upload-Metadata: name $(basename "${file}" | tr -d '\n' | base64)" \ + --header "Upload-Length: $size" \ + --header "Upload-Metadata: name $(basename "$file" | tr -d '\n' | base64)" \ --header "Connection: close" \ - ${TUSD} | grep 'Location:' | awk '{print $2}' | tr -d '\015' 1> $tus_location 2>&1 & - return $? + "$TUSD" | grep -i '^Location:' | sed -E 's/^[^:]+:\s*(.*)$/\1/' | tr -d '\r' } function tus_upload { # trap 'kill $(jobs -p)' SIGINT SIGTERM - local dest="${1}" - local file="${2}" - local size="${3}" - local parts="${4}" - local multi="${5}" - local color=$(( $RANDOM % 7 + 31 )) - - response=$(mktemp -u) - mkfifo $response + local dest="$1" + local file="$2" + local size="$3" + local parts="$4" + local multi="$5" + local color=$(( RANDOM % 7 + 31 )) while true; do local part=0; local finished=1 - while [ "${part}" -lt "${parts}" ]; do - if ! tus_complete "${file}" "${dest}" "${part}"; then + while [ "$part" -lt "$parts" ]; do + if ! tus_complete "$file" "$dest" "$part"; then local offset=$(( part * TUS_CHUNK_SIZE )) local file_offset=$(( (multi - 1) * parts * TUS_CHUNK_SIZE )) local length=$(( size - file_offset - offset )) - [ ${length} -gt "${TUS_CHUNK_SIZE}" ] \ - && length="${TUS_CHUNK_SIZE}" - - echo -e "\033[0;32m🠹\033[0m Upload \033[1m\e[1;${color}mpart ${part}\e[0m\033[21m." - echo -e "\033[2m↳ ${dest}" - eval "local tus_headers=($(expand '-H' "${TUS_HEADERS[@]}"))" + if [ "$length" -gt "$TUS_CHUNK_SIZE" ]; then + length="$TUS_CHUNK_SIZE" + fi + + echo -e "\033[0;32m🠹\033[0m Upload \033[1m\e[1;${color}mpart $part\e[0m\033[0m." + echo -e "\033[2m↳ $dest" + local header_args=() + set_header_args "${TUS_HEADERS[@]}" # TODO curl will throw (55: Broken pipe) # >> tusd will break connection after 'Content-Length' # >> ? curl: how to set content length to upload + response="$( curl --progress-bar \ - "${tus_headers[@]}" \ + "${header_args[@]}" \ --write-out "%{http_code}" \ --request PATCH \ --header "Tus-Resumable: 1.0.0" \ --header "Content-Type: application/offset+octet-stream" \ - --header "Content-Length: ${length}" \ - --header "Upload-Offset: ${offset}" \ + --header "Content-Length: $length" \ + --header "Upload-Offset: $offset" \ --data-binary "@-" \ - "${dest}" < <(dd if=${file} skip=$(( file_offset + offset )) count=${length} iflag=skip_bytes,count_bytes 2>/dev/null) 1> $response & + "$dest" < <(dd if="$file" skip=$(( file_offset + offset )) count="$length" iflag=skip_bytes,count_bytes status=none) + )" - tus_complete "${file}" "${dest}" "${part}" "$(tee < $response)" - echo -ne "\033[22m" + tus_complete "$file" "$dest" "$part" "$response" + echo -ne "\033[0m" finished=0 fi part=$((part + 1)) done - [ ${finished} -eq 1 ] && break + [ "$finished" = 1 ] && break done - rm $response } # TODO unhandled error codes # - 409: mismatched offset # - 423: file currently locked function tus_complete { - local file="${1}" - local dest="${2}" - local part="${3}" - local -r code="$(echo ${4} | grep -o '[[:digit:]]*$')" - local -r key="# 🠹 ${dest} [${part}]" + local file="$1" + local dest="$2" + local part="$3" + local -r code="$(echo "$4" | grep -o '[[:digit:]]*$')" + local -r key="# 🠹 $dest [$part]" - if [ -z "${code}" ]; then + if [ -z "$code" ]; then # read - cat "${file}.tus" | grep -qF "${key}" + grep -qF "$key" "$file.tus" return $? - elif [ "${code}" -eq 204 ] || [ "${code}" -eq 100 ]; then + elif [ "$code" = 204 ] || [ "$code" = 100 ]; then # TODO what to do with status_code 100 Continue? # write - echo "${key} uploaded." >> "${FILE}.tus" + echo "$key uploaded." >> "$FILE.tus" else - warning "\033[1m${part}\033[21m failed. Will retry.\n\033[2m↳ $(echo ${4} | tr -d '\n')\033[22m" + warning "\033[1m$part\033[0m failed. Will retry.\n\033[2m↳ $(echo "$4" | tr -d '\n')\033[0m" fi } function tus_concat { - local file="${1}" + local file="$1" - eval "local tus_headers=($(expand '-H' "${TUS_HEADERS[@]}"))" + local header_args=() + set_header_args "${TUS_HEADERS[@]}" curl -v \ - "${tus_headers[@]}" \ + --fail-with-body \ + "${header_args[@]}" \ --head \ --request POST \ --header "Tus-Resumable: 1.0.0" \ --header "Content-Length: 0" \ - --header "Upload-Concat: final;$(strip-domain "${TUSD}" "${@}")" \ - --header "Upload-Metadata: name $(basename "${file}" | tr -d '\n' | base64)" \ + --header "Upload-Concat: final;$(strip-domain "$TUSD" "$@")" \ + --header "Upload-Metadata: name $(basename "$file" | tr -d '\n' | base64)" \ --header "Connection: close" \ - ${TUSD} | grep 'Location:' | awk '{print $2}' | tr -d '\015' 1> $tus_location 2>&1 & - echo $? + "$TUSD" | grep -i '^Location:' | sed -E 's/^[^:]+:\s*(.*)$/\1/' | tr -d '\r' } function tus_done { - local dest="${1}" + local dest="$1" echo -e "\n\033[0;32m➤ All parts uploaded 🐈\033[0m" - [ -n "${dest}" ] && echo -e "\033[2m↳ ${dest}\033[22m" + [ -n "$dest" ] && echo -e "\033[2m↳ $dest\033[0m" exit 0 } @@ -218,38 +235,33 @@ function tus_done { # main declare -A pids -function cleanup { - rm $tus_location - # kill "${pids[@]}" -} -trap cleanup EXIT - # ============================================================================== # pre -IFS=',' read -r -a TUS_HEADERS <<< "${TUS_HEADERS}" +IFS=',' read -r -a TUS_HEADERS <<< "$TUS_HEADERS" OPT_RESET=0 OPT_OPTIONS=0 -while getopts ohrc:p:H:t: opt +OPT_DEBUG=0 +while getopts ohrc:p:H:t:D opt do - case "${opt}" in + case "$opt" in t) - TUSD="${OPTARG}" + TUSD="$OPTARG" ;; c) - [ -z "${OPTARG}" ] || echo "${OPTARG}" | grep -vq '^[[:digit:]]*$' \ + [ -z "$OPTARG" ] || echo "$OPTARG" | grep -vq '^[[:digit:]]*$' \ && usage TUS_CHUNK_SIZE=$((OPTARG * 1048576)) ;; p) - echo "${OPTARG}" | grep -vq '^[[:digit:]]*$' \ + echo "$OPTARG" | grep -vq '^[[:digit:]]*$' \ && usage - TUS_PARALLEL="${OPTARG}" + TUS_PARALLEL="$OPTARG" - [ ${TUS_PARALLEL} -gt 1 ] \ + [ "$TUS_PARALLEL" -gt 1 ] \ && tus_supports "concatenation" "Tusd can not perform parallel uploads" ;; H) - TUS_HEADERS+=("${OPTARG}") + TUS_HEADERS+=("$OPTARG") ;; r) OPT_RESET=1 @@ -257,23 +269,36 @@ do o) OPT_OPTIONS=1 ;; + D) + OPT_DEBUG=1 + ;; ?) usage esac done shift $((OPTIND - 1)) -[ -z "${TUSD}" ] && error 'tusd endpoint' 'TUSD' -readarray -t TUS_OPTIONS <<<"$(tus_options)" -if [ "${OPT_OPTIONS}" -eq 1 ]; then - echo -e "\033[1;34m➤ ${TUSD} ☁\033[0m\n\033[2m⌄\033[22m" - if [ -z "${TUS_OPTIONS[*]}" ]; then +if [ $OPT_DEBUG = 1 ]; then + set -x +fi + +[ -z "$TUSD" ] && error 'tusd endpoint' 'TUSD' +if ! tus_options="$(get_tus_options)"; then + echo "error: failed to get TUS options:" + echo "$tus_options" + exit 1 +fi +readarray -t TUS_OPTIONS <<<"$tus_options" +if [ "$OPT_OPTIONS" = 1 ]; then + echo -e "\033[1;34m➤ $TUSD ☁\033[0m\n\033[2m⌄\033[0m" + if [ ${#TUS_OPTIONS[@]} = 0 ]; then error "No tusd headers" else for opt in "${TUS_OPTIONS[@]}"; do - IFS=':' read -r -a header <<< "${opt}" - echo -e "\033[0;32m➤\033[0m ${header[0]}" - IFS=',' read -r -a values <<< "${header[1]}" + IFS=':' read -r key values <<< "$opt" + echo -e "\033[0;32m➤\033[0m $key" + if [ "${values:0:1}" = " " ]; then values="${values:1}"; fi + IFS=',' read -r -a values <<< "$values" for value in "${values[@]}"; do - echo -e "\033[2m↳ $(echo "${value}" | xargs)\033[22m" + echo -e "\033[2m ↳ $(echo "$value" | xargs)\033[0m" done done; fi exit 0 @@ -281,71 +306,78 @@ fi tus_supports "creation" "Tusd can not perform file creation" -[ ! -f "${1}" ] && usage 'No file given' -FILE="${1}" -SIZE="$(wc -c <"${FILE}")" +[ ! -f "$1" ] && usage 'No file given' +FILE="$1" +SIZE="$(wc -c <"$FILE")" -[ "${OPT_RESET}" -eq 1 ] \ - && rm "${FILE}.tus" &>/dev/null || true +if [ "$OPT_RESET" = 1 ]; then + rm "$FILE.tus" &>/dev/null +fi declare -A Location # shellcheck source=/dev/null -[ -f "${FILE}.tus" ] \ - && . <(cat "${FILE}.tus" | grep -E '^(TUSD|TUS_HEADERS|TUS_CHUNK_SIZE|TUS_PARALLEL|Location)') +if [ -f "$FILE.tus" ]; then + source <(grep -E '^(TUSD|TUS_HEADERS|TUS_CHUNK_SIZE|TUS_PARALLEL|Location)=' "$FILE.tus") +fi # TODO vaildate values. -total_parts="$(ceil "${SIZE}" "${TUS_CHUNK_SIZE}")" +total_parts="$(ceildiv "$SIZE" "$TUS_CHUNK_SIZE")" -if [ -z "${Location[@]}" ]; then +if [ ${#Location[@]} = 0 ]; then headers=() - [ "${TUS_PARALLEL}" -gt 1 ] \ - && headers+=("Upload-Concat: partial") + if [ "$TUS_PARALLEL" -gt 1 ]; then + headers+=("Upload-Concat: partial") + fi parts_per_partial=$((total_parts / TUS_PARALLEL)) - for i in $(seq ${TUS_PARALLEL}); do - partials="${parts_per_partial}" - if [ "${i}" -eq "${TUS_PARALLEL}" ]; then + for i in $(seq "$TUS_PARALLEL"); do + partials="$parts_per_partial" + if [ "$i" = "$TUS_PARALLEL" ]; then partials=$(( partials + total_parts - i * parts_per_partial )) length=$(( SIZE - ( TUS_PARALLEL - 1 ) * parts_per_partial * TUS_CHUNK_SIZE )) else length=$(( partials * TUS_CHUNK_SIZE )) fi - tus_create "${FILE}" "${length}" "${headers[@]}" || error 'Tus file creation failed' - read dest < $tus_location - Location[$dest]="${partials}-${i}" + if ! dest="$(tus_create "$FILE" "$length" "${headers[@]}")"; then + error 'Tus file creation failed' + fi + # FIXME handle empty destinations + # if [ -z "$dest" ]; then + Location[$dest]="$partials-$i" done # create resumable log file - cat <<-EOTUS > "${FILE}.tus" + cat <<-EOTUS > "$FILE.tus" # settings -TUSD="${TUSD}" +TUSD="$TUSD" TUS_HEADERS="$(join ', ' "${TUS_HEADERS[@]}")" -TUS_CHUNK_SIZE="${TUS_CHUNK_SIZE}" -TUS_PARALLEL="${TUS_PARALLEL}" +TUS_CHUNK_SIZE="$TUS_CHUNK_SIZE" +TUS_PARALLEL="$TUS_PARALLEL" # tus parts $(declare -p Location) -# ∑ ${total_parts} parts to upload. +# ∑ $total_parts parts to upload. EOTUS fi -echo -e "\033[0;35m∑\033[0m \033[1m${total_parts} parts\033[21m to upload." +echo -e "\033[0;35m∑\033[0m \033[1m$total_parts parts\033[0m to upload." for dest in "${!Location[@]}"; do - __=(${Location[$dest]//-/ }) - tus_upload "${dest}" "${FILE}" "${SIZE}" "${__[0]}" "${__[1]}" & - pids[$dest]=${!} + read parts multi <<<"${Location[$dest]//-/ }" + tus_upload "$dest" "$FILE" "$SIZE" "$parts" "$multi" & + pids[$dest]=$! done while [ ${#pids[@]} -gt 0 ]; do for dest in "${!Location[@]}"; do [ -e "/proc/${pids[$dest]}" ] && continue - unset pids[$dest] + unset "pids[$dest]" done sleep 1 done -if [ "${TUS_PARALLEL}" -gt 1 ]; then - tus_concat "${!Location[@]}" || error 'Tus concatenation failed' - read dest < $tus_location +if [ "$TUS_PARALLEL" -gt 1 ]; then + if ! dest="$(tus_concat "${!Location[@]}")"; then + error 'Tus concatenation failed' + fi fi -tus_done "${dest}" +tus_done "$dest"