Skip to content

Commit

Permalink
gupload: Show failed files or log it to a file | Fix #121
Browse files Browse the repository at this point in the history 
If upload failed for some files then

  If running in terminal, then

      if failed files > 20, then

        print to terminal

      else

        save to a log file, with a random name

  If not running in a terminal, then

      print all
  • Loading branch information
@Akianonymus
Akianonymus committed Nov 11, 2020
1 parent 59fa888 commit a4f47f3
Show file tree
Hide file tree
Showing 6 changed files with 132 additions and 32 deletions.
41 changes: 33 additions & 8 deletions bash/release/gupload
View file
Original file line number Diff line number Diff line change
Expand Up @@ -873,7 +873,7 @@ _upload_file_main() {
continue
done
[[ -n ${4} ]] && {
{ [[ ${RETURN_STATUS} = 1 ]] && printf "%s\n" "${RETURN_STATUS}"; } || printf "%s\n" "${RETURN_STATUS}" 1>&2
{ [[ ${RETURN_STATUS} = 1 ]] && printf "%s\n" "${file}"; } || printf "%s\n" "${file}" 1>&2
}
return 0
}
Expand All @@ -893,13 +893,15 @@ _upload_file_main() {
_upload_folder() {
[[ $# -lt 3 ]] && printf "%s: Missing arguments\n" "${FUNCNAME[0]}" && return 1
declare mode="${1}" files="${3}" && PARSE_MODE="${2}" ID="${4:-}" && export PARSE_MODE ID
unset SUCCESS_STATUS SUCCESS_FILES ERROR_STATUS ERROR_FILES
case "${mode}" in
normal)
[[ ${PARSE_MODE} = parse ]] && _clear_line 1 && _newline "\n"

while read -u 4 -r file; do
_upload_file_main "${PARSE_MODE}" "${file}" "${ID}"
: "$((RETURN_STATUS < 2 ? (SUCCESS_STATUS += 1) : (ERROR_STATUS += 1)))"
{ [[ ${RETURN_STATUS} = 1 ]] && : "$((SUCCESS_STATUS += 1))" && SUCCESS_FILES+="${file}"$'\n'; } ||
{ : "$((ERROR_STATUS += 1))" && ERROR_FILES+="${file}"$'\n'; }
if [[ -n ${VERBOSE:-${VERBOSE_PROGRESS}} ]]; then
_print_center "justify" "Status: ${SUCCESS_STATUS} Uploaded" " | ${ERROR_STATUS} Failed" "=" && _newline "\n"
else
Expand Down Expand Up @@ -931,8 +933,8 @@ _upload_folder() {
_clear_line 1 && "${QUIET:-_print_center}" "justify" "Status" ": ${SUCCESS_STATUS} Uploaded | ${ERROR_STATUS} Failed" "="
TOTAL="$((SUCCESS_STATUS + ERROR_STATUS))"
done
SUCCESS_STATUS="$(_count < "${TMPFILE}"SUCCESS)"
ERROR_STATUS="$(_count < "${TMPFILE}"ERROR)"
SUCCESS_STATUS="$(_count < "${TMPFILE}"SUCCESS)" SUCCESS_FILES="$(< "${TMPFILE}"SUCCESS)"
ERROR_STATUS="$(_count < "${TMPFILE}"ERROR)" ERROR_FILES="$(< "${TMPFILE}"ERROR)"
;;
esac
return 0
Expand Down Expand Up @@ -1551,13 +1553,36 @@ _process_arguments() {
if [[ ${EMPTY} != 1 ]]; then
[[ -z ${VERBOSE:-${VERBOSE_PROGRESS}} ]] && for _ in 1 2; do _clear_line 1; done

[[ ${SUCCESS_STATUS} -gt 0 ]] &&
FOLDER_ID="$(: "${DIRIDS%%$'\n'*}" && printf "%s\n" "${_/"|:_//_:|"*/}")" &&
_share_and_print_link "${FOLDER_ID}"
FOLDER_ID="$(: "${DIRIDS%%$'\n'*}" && printf "%s\n" "${_/"|:_//_:|"*/}")"

[[ ${SUCCESS_STATUS} -gt 0 ]] && _share_and_print_link "${FOLDER_ID}"

_newline "\n"
[[ ${SUCCESS_STATUS} -gt 0 ]] && "${QUIET:-_print_center}" "justify" "Total Files " "Uploaded: ${SUCCESS_STATUS}" "="
[[ ${ERROR_STATUS} -gt 0 ]] && "${QUIET:-_print_center}" "justify" "Total Files " "Failed: ${ERROR_STATUS}" "="
[[ ${ERROR_STATUS} -gt 0 ]] && "${QUIET:-_print_center}" "justify" "Total Files " "Failed: ${ERROR_STATUS}" "=" && {
# If running inside a terminal, then check if failed files are more than 25, if not, then print, else save in a log file
if [[ -t 1 ]]; then
{ [[ ${ERROR_STATUS} -le 25 ]] && printf "%s\n" "${ERROR_FILES}"; } || {
epoch_time="$(printf "%(%s)T\\n" "-1")" log_file_name="${0##*/}_${FOLDER_NAME}_${epoch_time}.failed"
# handle in case the vivid random file name was already there
i=0 && until ! [[ -f ${log_file_name} ]]; do
: $((i += 1)) && log_file_name="${0##*/}_${FOLDER_NAME}_$((epoch_time + i)).failed"
done
printf "%s\n%s\n%s\n\n%s\n%s\n" \
"Folder name: ${FOLDER_NAME} | Folder ID: ${FOLDER_ID}" \
"Run this command to retry the failed uploads:" \
" ${0##*/} --skip-duplicates \"${input}\" --root-dir \"${NEXTROOTDIRID}\" ${SKIP_SUBDIRS:+-s} ${PARALLEL_UPLOAD:+--parallel} ${PARALLEL_UPLOAD:+${NO_OF_PARALLEL_JOBS}}" \
"Failed files:" \
"${ERROR_FILES}" >> "${log_file_name}"
printf "%s\n" "To see the failed files, open \"${log_file_name}\""
printf "%s\n" "To retry the failed uploads only, use -d / --skip-duplicates flag. See log file for more help."

}
# if not running inside a terminal, print it all
else
printf "%s\n" "${ERROR_FILES}"
fi
}
printf "\n"
else
for _ in 1 2 3; do _clear_line 1; done
Expand Down
10 changes: 6 additions & 4 deletions bash/upload-utils.bash
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ _upload_file_main() {
continue
done
[[ -n ${4} ]] && {
{ [[ ${RETURN_STATUS} = 1 ]] && printf "%s\n" "${RETURN_STATUS}"; } || printf "%s\n" "${RETURN_STATUS}" 1>&2
{ [[ ${RETURN_STATUS} = 1 ]] && printf "%s\n" "${file}"; } || printf "%s\n" "${file}" 1>&2
}
return 0
}
Expand All @@ -121,13 +121,15 @@ _upload_file_main() {
_upload_folder() {
[[ $# -lt 3 ]] && printf "%s: Missing arguments\n" "${FUNCNAME[0]}" && return 1
declare mode="${1}" files="${3}" && PARSE_MODE="${2}" ID="${4:-}" && export PARSE_MODE ID
unset SUCCESS_STATUS SUCCESS_FILES ERROR_STATUS ERROR_FILES
case "${mode}" in
normal)
[[ ${PARSE_MODE} = parse ]] && _clear_line 1 && _newline "\n"

while read -u 4 -r file; do
_upload_file_main "${PARSE_MODE}" "${file}" "${ID}"
: "$((RETURN_STATUS < 2 ? (SUCCESS_STATUS += 1) : (ERROR_STATUS += 1)))"
{ [[ ${RETURN_STATUS} = 1 ]] && : "$((SUCCESS_STATUS += 1))" && SUCCESS_FILES+="${file}"$'\n'; } ||
{ : "$((ERROR_STATUS += 1))" && ERROR_FILES+="${file}"$'\n'; }
if [[ -n ${VERBOSE:-${VERBOSE_PROGRESS}} ]]; then
_print_center "justify" "Status: ${SUCCESS_STATUS} Uploaded" " | ${ERROR_STATUS} Failed" "=" && _newline "\n"
else
Expand Down Expand Up @@ -159,8 +161,8 @@ _upload_folder() {
_clear_line 1 && "${QUIET:-_print_center}" "justify" "Status" ": ${SUCCESS_STATUS} Uploaded | ${ERROR_STATUS} Failed" "="
TOTAL="$((SUCCESS_STATUS + ERROR_STATUS))"
done
SUCCESS_STATUS="$(_count < "${TMPFILE}"SUCCESS)"
ERROR_STATUS="$(_count < "${TMPFILE}"ERROR)"
SUCCESS_STATUS="$(_count < "${TMPFILE}"SUCCESS)" SUCCESS_FILES="$(< "${TMPFILE}"SUCCESS)"
ERROR_STATUS="$(_count < "${TMPFILE}"ERROR)" ERROR_FILES="$(< "${TMPFILE}"ERROR)"
;;
esac
return 0
Expand Down
31 changes: 27 additions & 4 deletions bash/upload.bash
View file
Original file line number Diff line number Diff line change
Expand Up @@ -613,13 +613,36 @@ _process_arguments() {
if [[ ${EMPTY} != 1 ]]; then
[[ -z ${VERBOSE:-${VERBOSE_PROGRESS}} ]] && for _ in 1 2; do _clear_line 1; done

[[ ${SUCCESS_STATUS} -gt 0 ]] &&
FOLDER_ID="$(: "${DIRIDS%%$'\n'*}" && printf "%s\n" "${_/"|:_//_:|"*/}")" &&
_share_and_print_link "${FOLDER_ID}"
FOLDER_ID="$(: "${DIRIDS%%$'\n'*}" && printf "%s\n" "${_/"|:_//_:|"*/}")"

[[ ${SUCCESS_STATUS} -gt 0 ]] && _share_and_print_link "${FOLDER_ID}"

_newline "\n"
[[ ${SUCCESS_STATUS} -gt 0 ]] && "${QUIET:-_print_center}" "justify" "Total Files " "Uploaded: ${SUCCESS_STATUS}" "="
[[ ${ERROR_STATUS} -gt 0 ]] && "${QUIET:-_print_center}" "justify" "Total Files " "Failed: ${ERROR_STATUS}" "="
[[ ${ERROR_STATUS} -gt 0 ]] && "${QUIET:-_print_center}" "justify" "Total Files " "Failed: ${ERROR_STATUS}" "=" && {
# If running inside a terminal, then check if failed files are more than 25, if not, then print, else save in a log file
if [[ -t 1 ]]; then
{ [[ ${ERROR_STATUS} -le 25 ]] && printf "%s\n" "${ERROR_FILES}"; } || {
epoch_time="$(printf "%(%s)T\\n" "-1")" log_file_name="${0##*/}_${FOLDER_NAME}_${epoch_time}.failed"
# handle in case the vivid random file name was already there
i=0 && until ! [[ -f ${log_file_name} ]]; do
: $((i += 1)) && log_file_name="${0##*/}_${FOLDER_NAME}_$((epoch_time + i)).failed"
done
printf "%s\n%s\n%s\n\n%s\n%s\n" \
"Folder name: ${FOLDER_NAME} | Folder ID: ${FOLDER_ID}" \
"Run this command to retry the failed uploads:" \
" ${0##*/} --skip-duplicates \"${input}\" --root-dir \"${NEXTROOTDIRID}\" ${SKIP_SUBDIRS:+-s} ${PARALLEL_UPLOAD:+--parallel} ${PARALLEL_UPLOAD:+${NO_OF_PARALLEL_JOBS}}" \
"Failed files:" \
"${ERROR_FILES}" >> "${log_file_name}"
printf "%s\n" "To see the failed files, open \"${log_file_name}\""
printf "%s\n" "To retry the failed uploads only, use -d / --skip-duplicates flag. See log file for more help."

}
# if not running inside a terminal, print it all
else
printf "%s\n" "${ERROR_FILES}"
fi
}
printf "\n"
else
for _ in 1 2 3; do _clear_line 1; done
Expand Down
41 changes: 33 additions & 8 deletions sh/release/gupload
View file
Original file line number Diff line number Diff line change
Expand Up @@ -855,7 +855,7 @@ _upload_file_main() {
continue
done
[ -n "${4}" ] && {
{ [ "${RETURN_STATUS}" = 1 ] && printf "%s\n" "${RETURN_STATUS}"; } || printf "%s\n" "${RETURN_STATUS}" 1>&2
{ [ "${RETURN_STATUS}" = 1 ] && printf "%s\n" "${file_upload_file_main}"; } || printf "%s\n" "${file_upload_file_main}" 1>&2
}
return 0
}
Expand All @@ -875,13 +875,15 @@ _upload_file_main() {
_upload_folder() {
[ $# -lt 3 ] && printf "Missing arguments\n" && return 1
mode_upload_folder="${1}" PARSE_MODE="${2}" files_upload_folder="${3}" ID="${4:-}" && export PARSE_MODE ID
unset SUCCESS_STATUS SUCCESS_FILES ERROR_STATUS ERROR_FILES
case "${mode_upload_folder}" in
normal)
[ "${PARSE_MODE}" = parse ] && _clear_line 1 && _newline "\n"
while read -r file <&4; do
_upload_file_main "${PARSE_MODE}" "${file}" "${ID}"
: "$((RETURN_STATUS < 2 ? (SUCCESS_STATUS += 1) : (ERROR_STATUS += 1)))"
{ [ "${RETURN_STATUS}" = 1 ] && : "$((SUCCESS_STATUS += 1))" && SUCCESS_FILES="$(printf "%b\n" "${SUCCESS_STATUS:+${SUCCESS_STATUS}\n}${file}")"; } ||
{ : "$((ERROR_STATUS += 1))" && ERROR_FILES="$(printf "%b\n" "${ERROR_STATUS:+${ERROR_STATUS}\n}${file}")"; }
if [ -n "${VERBOSE:-${VERBOSE_PROGRESS}}" ]; then
_print_center "justify" "Status: ${SUCCESS_STATUS} Uploaded" " | ${ERROR_STATUS} Failed" "=" && _newline "\n"
else
Expand Down Expand Up @@ -916,8 +918,9 @@ EOF
_clear_line 1 && "${QUIET:-_print_center}" "justify" "Status" ": ${SUCCESS_STATUS} Uploaded | ${ERROR_STATUS} Failed" "="
TOTAL="$((SUCCESS_STATUS + ERROR_STATUS))"
done
SUCCESS_STATUS="$(($(wc -l < "${TMPFILE}"SUCCESS)))"
ERROR_STATUS="$(($(wc -l < "${TMPFILE}"ERROR)))"
SUCCESS_STATUS="$(($(wc -l < "${TMPFILE}"SUCCESS)))" SUCCESS_FILES="$(cat "${TMPFILE}"SUCCESS)"
ERROR_STATUS="$(($(wc -l < "${TMPFILE}"ERROR)))" ERROR_FILES="$(cat "${TMPFILE}"ERROR)"
export SUCCESS_FILES ERROR_FILES
;;
esac
return 0
Expand Down Expand Up @@ -1552,13 +1555,35 @@ EOF
if [ "${EMPTY}" != 1 ]; then
[ -z "${VERBOSE:-${VERBOSE_PROGRESS}}" ] && for _ in 1 2; do _clear_line 1; done
[ "${SUCCESS_STATUS}" -gt 0 ] &&
FOLDER_ID="$(_tmp="$(printf "%s\n" "${DIRIDS}" | while read -r line; do printf "%s\n" "${line}" && break; done)" && printf "%s\n" "${_tmp%%"|:_//_:|"*}")" &&
_share_and_print_link "${FOLDER_ID}"
FOLDER_ID="$(_tmp="$(printf "%s\n" "${DIRIDS}" | while read -r line; do printf "%s\n" "${line}" && break; done)" && printf "%s\n" "${_tmp%%"|:_//_:|"*}")"
[ "${SUCCESS_STATUS}" -gt 0 ] && _share_and_print_link "${FOLDER_ID}"
_newline "\n"
[ "${SUCCESS_STATUS}" -gt 0 ] && "${QUIET:-_print_center}" "justify" "Total Files " "Uploaded: ${SUCCESS_STATUS}" "="
[ "${ERROR_STATUS}" -gt 0 ] && "${QUIET:-_print_center}" "justify" "Total Files " "Failed: ${ERROR_STATUS}" "="
[ "${ERROR_STATUS}" -gt 0 ] && "${QUIET:-_print_center}" "justify" "Total Files " "Failed: ${ERROR_STATUS}" "=" && {
# If running inside a terminal, then check if failed files are more than 25, if not, then print, else save in a log file
if [ -t 1 ]; then
{ [ "${ERROR_STATUS}" -le 25 ] && printf "%s\n" "${ERROR_FILES}"; } || {
epoch_time="$(date +'%s')" log_file_name="${0##*/}_${FOLDER_NAME}_${epoch_time}.failed"
# handle in case the vivid random file name was already there
i=0 && until ! [ -f "${log_file_name}" ]; do
: $((i += 1)) && log_file_name="${0##*/}_${FOLDER_NAME}_$((epoch_time + i)).failed"
done
printf "%s\n%s\n%s\n\n%s\n%s\n" \
"Folder name: ${FOLDER_NAME} | Folder ID: ${FOLDER_ID}" \
"Run this command to retry the failed uploads:" \
" ${0##*/} --skip-duplicates \"${input}\" --root-dir \"${NEXTROOTDIRID}\" ${SKIP_SUBDIRS:+-s} ${PARALLEL_UPLOAD:+--parallel} ${PARALLEL_UPLOAD:+${NO_OF_PARALLEL_JOBS}}" \
"Failed files:" \
"${ERROR_FILES}" >> "${log_file_name}"
printf "%s\n" "To see the failed files, open \"${log_file_name}\""
printf "%s\n" "To retry the failed uploads only, use -d / --skip-duplicates flag. See log file for more help."
}
# if not running inside a terminal, print it all
else
printf "%s\n" "${ERROR_FILES}"
fi
}
printf "\n"
else
for _ in 1 2 3; do _clear_line 1; done
Expand Down
11 changes: 7 additions & 4 deletions sh/upload-utils.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ _upload_file_main() {
continue
done
[ -n "${4}" ] && {
{ [ "${RETURN_STATUS}" = 1 ] && printf "%s\n" "${RETURN_STATUS}"; } || printf "%s\n" "${RETURN_STATUS}" 1>&2
{ [ "${RETURN_STATUS}" = 1 ] && printf "%s\n" "${file_upload_file_main}"; } || printf "%s\n" "${file_upload_file_main}" 1>&2
}
return 0
}
Expand All @@ -121,13 +121,15 @@ _upload_file_main() {
_upload_folder() {
[ $# -lt 3 ] && printf "Missing arguments\n" && return 1
mode_upload_folder="${1}" PARSE_MODE="${2}" files_upload_folder="${3}" ID="${4:-}" && export PARSE_MODE ID
unset SUCCESS_STATUS SUCCESS_FILES ERROR_STATUS ERROR_FILES
case "${mode_upload_folder}" in
normal)
[ "${PARSE_MODE}" = parse ] && _clear_line 1 && _newline "\n"

while read -r file <&4; do
_upload_file_main "${PARSE_MODE}" "${file}" "${ID}"
: "$((RETURN_STATUS < 2 ? (SUCCESS_STATUS += 1) : (ERROR_STATUS += 1)))"
{ [ "${RETURN_STATUS}" = 1 ] && : "$((SUCCESS_STATUS += 1))" && SUCCESS_FILES="$(printf "%b\n" "${SUCCESS_STATUS:+${SUCCESS_STATUS}\n}${file}")"; } ||
{ : "$((ERROR_STATUS += 1))" && ERROR_FILES="$(printf "%b\n" "${ERROR_STATUS:+${ERROR_STATUS}\n}${file}")"; }
if [ -n "${VERBOSE:-${VERBOSE_PROGRESS}}" ]; then
_print_center "justify" "Status: ${SUCCESS_STATUS} Uploaded" " | ${ERROR_STATUS} Failed" "=" && _newline "\n"
else
Expand Down Expand Up @@ -162,8 +164,9 @@ EOF
_clear_line 1 && "${QUIET:-_print_center}" "justify" "Status" ": ${SUCCESS_STATUS} Uploaded | ${ERROR_STATUS} Failed" "="
TOTAL="$((SUCCESS_STATUS + ERROR_STATUS))"
done
SUCCESS_STATUS="$(($(wc -l < "${TMPFILE}"SUCCESS)))"
ERROR_STATUS="$(($(wc -l < "${TMPFILE}"ERROR)))"
SUCCESS_STATUS="$(($(wc -l < "${TMPFILE}"SUCCESS)))" SUCCESS_FILES="$(cat "${TMPFILE}"SUCCESS)"
ERROR_STATUS="$(($(wc -l < "${TMPFILE}"ERROR)))" ERROR_FILES="$(cat "${TMPFILE}"ERROR)"
export SUCCESS_FILES ERROR_FILES
;;
esac
return 0
Expand Down
30 changes: 26 additions & 4 deletions sh/upload.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -629,13 +629,35 @@ EOF
if [ "${EMPTY}" != 1 ]; then
[ -z "${VERBOSE:-${VERBOSE_PROGRESS}}" ] && for _ in 1 2; do _clear_line 1; done
[ "${SUCCESS_STATUS}" -gt 0 ] &&
FOLDER_ID="$(_tmp="$(printf "%s\n" "${DIRIDS}" | while read -r line; do printf "%s\n" "${line}" && break; done)" && printf "%s\n" "${_tmp%%"|:_//_:|"*}")" &&
_share_and_print_link "${FOLDER_ID}"
FOLDER_ID="$(_tmp="$(printf "%s\n" "${DIRIDS}" | while read -r line; do printf "%s\n" "${line}" && break; done)" && printf "%s\n" "${_tmp%%"|:_//_:|"*}")"
[ "${SUCCESS_STATUS}" -gt 0 ] && _share_and_print_link "${FOLDER_ID}"
_newline "\n"
[ "${SUCCESS_STATUS}" -gt 0 ] && "${QUIET:-_print_center}" "justify" "Total Files " "Uploaded: ${SUCCESS_STATUS}" "="
[ "${ERROR_STATUS}" -gt 0 ] && "${QUIET:-_print_center}" "justify" "Total Files " "Failed: ${ERROR_STATUS}" "="
[ "${ERROR_STATUS}" -gt 0 ] && "${QUIET:-_print_center}" "justify" "Total Files " "Failed: ${ERROR_STATUS}" "=" && {
# If running inside a terminal, then check if failed files are more than 25, if not, then print, else save in a log file
if [ -t 1 ]; then
{ [ "${ERROR_STATUS}" -le 25 ] && printf "%s\n" "${ERROR_FILES}"; } || {
epoch_time="$(date +'%s')" log_file_name="${0##*/}_${FOLDER_NAME}_${epoch_time}.failed"
# handle in case the vivid random file name was already there
i=0 && until ! [ -f "${log_file_name}" ]; do
: $((i += 1)) && log_file_name="${0##*/}_${FOLDER_NAME}_$((epoch_time + i)).failed"
done
printf "%s\n%s\n%s\n\n%s\n%s\n" \
"Folder name: ${FOLDER_NAME} | Folder ID: ${FOLDER_ID}" \
"Run this command to retry the failed uploads:" \
" ${0##*/} --skip-duplicates \"${input}\" --root-dir \"${NEXTROOTDIRID}\" ${SKIP_SUBDIRS:+-s} ${PARALLEL_UPLOAD:+--parallel} ${PARALLEL_UPLOAD:+${NO_OF_PARALLEL_JOBS}}" \
"Failed files:" \
"${ERROR_FILES}" >> "${log_file_name}"
printf "%s\n" "To see the failed files, open \"${log_file_name}\""
printf "%s\n" "To retry the failed uploads only, use -d / --skip-duplicates flag. See log file for more help."
}
# if not running inside a terminal, print it all
else
printf "%s\n" "${ERROR_FILES}"
fi
}
printf "\n"
else
for _ in 1 2 3; do _clear_line 1; done
Expand Down

0 comments on commit a4f47f3

Please sign in to comment.