[upload,sync].[sh,bash]: Add --include/-in and --exclude/-ex flag

exclude or include the files from uploading matching the given pattern upload script uses find, and gsync uses grep if -in/-ex is given with gsync in --arguments command, it will also work along with native option for gsync
labbots · Jul 26, 2020 · aaa8f4b · aaa8f4b
1 parent 9c964a9
commit aaa8f4b
Show file tree

Hide file tree

Showing 5 changed files with 115 additions and 26 deletions.
diff --git a/README.md b/README.md
@@ -115,7 +115,6 @@ This repo contains two types of scripts, posix compatible and bash compatible.
 
 | Program             | Role In Script                             |
 | ------------------- | ------------------------------------------ |
-| ps                  | For parallel upload progress               |
 | date                | For installation, update and Miscellaneous |
 | cat                 | Miscellaneous                              |
 | sleep               | Self explanatory                           |
@@ -446,6 +445,24 @@ These are the custom flags that are currently implemented:
 
     ---
 
+-   <strong>-in | --include 'pattern'</strong>
+
+    Only include the files with the given pattern to upload - Applicable for folder uploads.
+
+    e.g: gupload local_folder --include "*1*", will only include the files with pattern '1' in the name.
+
+    Note: Only provide patterns which are supported by find -name option.
+
+    ---
+
+-   <strong>-ex | --exclude 'pattern'</strong>
+
+    e.g: gupload local_folder --exclude "*1*", will exclude all the files with pattern '1' in the name.
+
+    Note: Only provide patterns which are supported by find -name option.
+
+    ---
+
 -   <strong>-q | --quiet</strong>
 
     Supress the normal output, only show success/error upload messages for files, and one extra line at the beginning for folder showing no. of files and sub folders.
@@ -683,6 +700,26 @@ Read this section thoroughly to fully utilise the sync script, feel free to open
 
     ---
 
+-   <strong>-in | --include 'pattern'</strong>
+
+    Only include the files with the given pattern to upload.
+
+    e.g: gsync local_folder --include "*1*", will only include the files with pattern '1' in the name.\n
+
+    Note: Only provide patterns which are supported by grep, and supported by -E option.
+
+    ---
+
+-   <strong>-ex | --exclude 'pattern'</strong>
+
+    Exclude the files with the given pattern from uploading.
+
+    e.g: gsync local_folder --exclude "*1*", will exclude all the files with pattern '1' in the name.\n
+
+    Note: Only provide patterns which are supported by grep, and supported by -E option.
+
+    ---
+
 -   <strong>-c | --command command_name</strong>
 
     Incase if gupload command installed with any other name or to use in systemd service, which requires fullpath.

diff --git a/bash/sync.bash b/bash/sync.bash
@@ -3,11 +3,11 @@
 # shellcheck source=/dev/null
 
 _usage() {
-    printf "
+    printf "%b" "
 The script can be used to sync your local folder to google drive.
 
 Utilizes google-drive-upload bash scripts.\n
-Usage: %s [options.. ]\n
+Usage: ${0##*/} [options.. ]\n
 Options:\n
   -d | --directory - Gdrive foldername.\n
   -k | --kill - to kill the background job using pid number ( -p flags ) or used with input, can be used multiple times.\n
@@ -16,18 +16,22 @@ Options:\n
   -p | --pid - Specify a pid number, used for --jobs or --kill or --info flags, can be used multiple times.\n
   -i | --info - See information about a specific sync using pid_number ( use -p flag ) or use with input, can be used multiple times.\n
   -t | --time <time_in_seconds> - Amount of time to wait before try to sync again in background.\n
-     To set wait time by default, use %s -t default='3'. Replace 3 with any positive integer.\n
+     To set wait time by default, use ${0##*/} -t default='3'. Replace 3 with any positive integer.\n
   -l | --logs - To show the logs after starting a job or show log of existing job. Can be used with pid number ( -p flag ).
      Note: If multiple pid numbers or inputs are used, then will only show log of first input as it goes on forever.
-  -a | --arguments - Additional arguments for gupload commands. e.g: %s -a '-q -o -p 4 -d'.\n
-     To set some arguments by default, use %s -a default='-q -o -p 4 -d'.\n
+  -a | --arguments - Additional arguments for gupload commands. e.g: ${0##*/} -a '-q -o -p 4 -d'.\n
+     To set some arguments by default, use ${0##*/} -a default='-q -o -p 4 -d'.\n
   -fg | --foreground - This will run the job in foreground and show the logs.\n
+  -in | --include 'pattern' - Only include the files with the given pattern to upload.\n
+       e.g: ${0##*/} local_folder --include "*1*", will only include with files with pattern '1' in the name.\n
+  -ex | --exclude 'pattern' - Exclude the files with the given pattern from uploading.\n
+       e.g: ${0##*/} local_folder --exclude "*1*", will exclude all files with pattern '1' in the name.\n
   -c | --command 'command name'- Incase if gupload command installed with any other name or to use in systemd service.\n
   --sync-detail-dir 'dirname' - Directory where a job information will be stored.
      Default: ${HOME}/.google-drive-upload\n
   -s | --service 'service name' - To generate systemd service file to setup background jobs on boot.\n
   -D | --debug - Display script command trace, use before all the flags to see maximum script trace.\n
-  -h | --help - Display usage instructions.\n" "${0##*/}" "${0##*/}" "${0##*/}" "${0##*/}"
+  -h | --help - Display usage instructions.\n"
     exit 0
 }
 
@@ -206,8 +210,10 @@ _check_and_upload() {
     [[ $(printf "%b\n" ./*) = "./*" ]] && return 0
 
     all+=(*)
-
-    mapfile -t new_files <<< "$(grep -vxFf <(printf "%s\n" "${initial[@]}") <(printf "%s\n" "${all[@]}") || :)"
+    # shellcheck disable=SC2086
+    { [ -n "${INCLUDE_FILES}" ] && mapfile -t all <<< "$(printf "%s\n" "${all[@]}" | grep -E ${INCLUDE_FILES})"; } || :
+    # shellcheck disable=SC2086
+    mapfile -t new_files <<< "$(eval grep -vxEf <(printf "%s\n" "${initial[@]}") <(printf "%s\n" "${all[@]}") ${EXCLUDE_FILES} || :)"
 
     [[ -n ${new_files[*]} ]] && printf "" >| "${ERROR_LOG}" && {
         declare -A Aseen && for new_file in "${new_files[@]}"; do
@@ -426,6 +432,14 @@ _setup_arguments() {
                 ARGS+="${2/default=/} " && shift
                 ;;
             -fg | --foreground) FOREGROUND="true" && SHOW_LOGS="true" ;;
+            -in | --include)
+                _check_longoptions "${1}" "${2}"
+                INCLUDE_FILES="${INCLUDE_FILES} -e '${2}' " && shift
+                ;;
+            -ex | --exclude)
+                _check_longoptions "${1}" "${2}"
+                EXCLUDE_FILES="${EXCLUDE_FILES} -e '${2}' " && shift
+                ;;
             -c | --command)
                 _check_longoptions "${1}" "${2}"
                 CUSTOM_COMMAND_NAME="${2}" && shift

diff --git a/bash/upload.bash b/bash/upload.bash
@@ -3,9 +3,9 @@
 # shellcheck source=/dev/null
 
 _usage() {
-    printf "
+    printf "%b" "
 The script can be used to upload file/directory to google drive.\n
-Usage:\n %s [options.. ] <filename> <foldername>\n
+Usage:\n ${0##*/} [options.. ] <filename> <foldername>\n
 Foldername argument is optional. If not provided, the file will be uploaded to preconfigured google drive.\n
 File name argument is optional if create directory option is used.\n
 Options:\n
@@ -22,16 +22,20 @@ Options:\n
   --speed 'speed' - Limit the download speed, supported formats: 1K, 1M and 1G.\n
   -i | --save-info <file_to_save_info> - Save uploaded files info to the given filename.\n
   -z | --config <config_path> - Override default config file with custom config file.\nIf you want to change default value, then use this format -z/--config default=default=your_config_file_path.\n
-  -R | --retry 'num of retries' - Retry the file upload if it fails, postive integer as argument. Currently only for file uploads.\n
   -q | --quiet - Supress the normal output, only show success/error upload messages for files, and one extra line at the beginning for folder showing no. of files and sub folders.\n
+  -R | --retry 'num of retries' - Retry the file upload if it fails, postive integer as argument. Currently only for file uploads.\n
+  -in | --include 'pattern' - Only include the files with the given pattern to upload - Applicable for folder uploads.\n
+      e.g: ${0##*/} local_folder --include "*1*", will only include with files with pattern '1' in the name.\n
+  -ex | --exclude 'pattern' - Exclude the files with the given pattern from uploading. - Applicable for folder uploads.\n
+      e.g: ${0##*/} local_folder --exclude "*1*", will exclude all the files pattern '1' in the name.\n
   -v | --verbose - Display detailed message (only for non-parallel uploads).\n
   -V | --verbose-progress - Display detailed message and detailed upload progress(only for non-parallel uploads).\n
   --skip-internet-check - Do not check for internet connection, recommended to use in sync jobs.\n
   -u | --update - Update the installed script in your system.\n
   --info - Show detailed info, only if script is installed system wide.\n
   -U | --uninstall - Uninstall script, remove related files.\n
   -D | --debug - Display script command trace.\n
-  -h | --help - Display usage instructions.\n" "${0##*/}"
+  -h | --help - Display usage instructions.\n"
     exit 0
 }
 
@@ -236,6 +240,14 @@ _setup_arguments() {
                     exit 1
                 fi
                 ;;
+            -in | --include)
+                _check_longoptions "${1}" "${2}"
+                INCLUDE_FILES="${INCLUDE_FILES} -name '${2}' " && shift
+                ;;
+            -ex | --exclude)
+                _check_longoptions "${1}" "${2}"
+                EXCLUDE_FILES="${EXCLUDE_FILES} ! -name '${2}' " && shift
+                ;;
             -q | --quiet) QUIET="_print_center_quiet" ;;
             -v | --verbose) VERBOSE="true" ;;
             -V | --verbose-progress) VERBOSE_PROGRESS="true" && CURL_PROGRESS="" ;;

diff --git a/sh/sync.sh b/sh/sync.sh
@@ -3,11 +3,11 @@
 # shellcheck source=/dev/null
 
 _usage() {
-    printf "
+    printf "%b" "
 The script can be used to sync your local folder to google drive.
 
 Utilizes google-drive-upload bash scripts.\n
-Usage: %s [options.. ]\n
+Usage: ${0##*/} [options.. ]\n
 Options:\n
   -d | --directory - Gdrive foldername.\n
   -k | --kill - to kill the background job using pid number ( -p flags ) or used with input, can be used multiple times.\n
@@ -16,18 +16,22 @@ Options:\n
   -p | --pid - Specify a pid number, used for --jobs or --kill or --info flags, can be used multiple times.\n
   -i | --info - See information about a specific sync using pid_number ( use -p flag ) or use with input, can be used multiple times.\n
   -t | --time <time_in_seconds> - Amount of time to wait before try to sync again in background.\n
-     To set wait time by default, use %s -t default='3'. Replace 3 with any positive integer.\n
+     To set wait time by default, use ${0##*/} -t default='3'. Replace 3 with any positive integer.\n
   -l | --logs - To show the logs after starting a job or show log of existing job. Can be used with pid number ( -p flag ).
      Note: If multiple pid numbers or inputs are used, then will only show log of first input as it goes on forever.
-  -a | --arguments - Additional arguments for gupload commands. e.g: %s -a '-q -o -p 4 -d'.\n
-     To set some arguments by default, use %s -a default='-q -o -p 4 -d'.\n
+  -a | --arguments - Additional arguments for gupload commands. e.g: ${0##*/} -a '-q -o -p 4 -d'.\n
+     To set some arguments by default, use ${0##*/} -a default='-q -o -p 4 -d'.\n
   -fg | --foreground - This will run the job in foreground and show the logs.\n
+  -in | --include 'pattern' - Only include the files with the given pattern to upload.\n
+       e.g: ${0##*/} local_folder --include "*1*", will only include with files with pattern '1' in the name.\n
+  -ex | --exclude 'pattern' - Exclude the files with the given pattern from uploading.\n
+       e.g: ${0##*/} local_folder --exclude "*1*", will exclude all files with pattern '1' in the name.\n
   -c | --command 'command name'- Incase if gupload command installed with any other name or to use in systemd service.\n
   --sync-detail-dir 'dirname' - Directory where a job information will be stored.
      Default: ${HOME}/.google-drive-upload\n
   -s | --service 'service name' - To generate systemd service file to setup background jobs on boot.\n
   -D | --debug - Display script command trace, use before all the flags to see maximum script trace.\n
-  -h | --help - Display usage instructions.\n" "${0##*/}" "${0##*/}" "${0##*/}" "${0##*/}"
+  -h | --help - Display usage instructions.\n"
     exit 0
 }
 
@@ -207,8 +211,9 @@ _check_and_upload() {
     # check if folder is empty
     [ "$(printf "%b\n" ./*)" = "./*" ] && return 0
 
+    # shellcheck disable=SC2086
     all_check_and_upload="${all_check_and_upload}
-$(printf -- "%b\n" *)"
+$(_tmp='printf -- "%b\n" * '${INCLUDE_FILES:+| grep -E ${INCLUDE_FILES}}'' && eval "${_tmp}")"
 
     # Open file discriptors for grep
     exec 5<< EOF
@@ -217,7 +222,8 @@ EOF
     exec 6<< EOF
 $(printf "%s\n" "${all_check_and_upload}")
 EOF
-    new_files_check_and_upload="$(grep -vxFf /dev/fd/5 /dev/fd/6 -e '^$' || :)"
+    # shellcheck disable=SC2086
+    new_files_check_and_upload="$(eval grep -vExf /dev/fd/5 /dev/fd/6 -e '^$' ${EXCLUDE_FILES} || :)"
     # close file discriptos
     exec 5<&- && exec 6<&-
 
@@ -463,6 +469,14 @@ _setup_arguments() {
                 ARGS=" ${ARGS} ${2##default=} " && shift
                 ;;
             -fg | --foreground) FOREGROUND="true" && SHOW_LOGS="true" ;;
+            -in | --include)
+                _check_longoptions "${1}" "${2}"
+                INCLUDE_FILES="${INCLUDE_FILES} -e '${2}' " && shift
+                ;;
+            -ex | --exclude)
+                _check_longoptions "${1}" "${2}"
+                EXCLUDE_FILES="${EXCLUDE_FILES} -e '${2}' " && shift
+                ;;
             -c | --command)
                 _check_longoptions "${1}" "${2}"
                 CUSTOM_COMMAND_NAME="${2}" && shift

diff --git a/sh/upload.sh b/sh/upload.sh
@@ -3,9 +3,9 @@
 # shellcheck source=/dev/null
 
 _usage() {
-    printf "
+    printf "%b" "
 The script can be used to upload file/directory to google drive.\n
-Usage:\n %s [options.. ] <filename> <foldername>\n
+Usage:\n ${0##*/} [options.. ] <filename> <foldername>\n
 Foldername argument is optional. If not provided, the file will be uploaded to preconfigured google drive.\n
 File name argument is optional if create directory option is used.\n
 Options:\n
@@ -24,14 +24,18 @@ Options:\n
   -z | --config <config_path> - Override default config file with custom config file.\nIf you want to change default value, then use this format -z/--config default=default=your_config_file_path.\n
   -q | --quiet - Supress the normal output, only show success/error upload messages for files, and one extra line at the beginning for folder showing no. of files and sub folders.\n
   -R | --retry 'num of retries' - Retry the file upload if it fails, postive integer as argument. Currently only for file uploads.\n
+  -in | --include 'pattern' - Only include the files with the given pattern to upload - Applicable for folder uploads.\n
+      e.g: ${0##*/} local_folder --include "*1*", will only include with files with pattern '1' in the name.\n
+  -ex | --exclude 'pattern' - Exclude the files with the given pattern from uploading. - Applicable for folder uploads.\n
+      e.g: ${0##*/} local_folder --exclude "*1*", will exclude all the files pattern '1' in the name.\n
   -v | --verbose - Display detailed message (only for non-parallel uploads).\n
   -V | --verbose-progress - Display detailed message and detailed upload progress(only for non-parallel uploads).\n
   --skip-internet-check - Do not check for internet connection, recommended to use in sync jobs.\n
   -u | --update - Update the installed script in your system.\n
   --info - Show detailed info, only if script is installed system wide.\n
   -U | --uninstall - Uninstall script, remove related files.\n
   -D | --debug - Display script command trace.\n
-  -h | --help - Display usage instructions.\n" "${0##*/}"
+  -h | --help - Display usage instructions.\n"
     exit 0
 }
 
@@ -249,6 +253,14 @@ _setup_arguments() {
                     exit 1
                 fi
                 ;;
+            -in | --include)
+                _check_longoptions "${1}" "${2}"
+                INCLUDE_FILES="${INCLUDE_FILES} -name '${2}' " && shift
+                ;;
+            -ex | --exclude)
+                _check_longoptions "${1}" "${2}"
+                EXCLUDE_FILES="${EXCLUDE_FILES} ! -name '${2}' " && shift
+                ;;
             -q | --quiet) QUIET="_print_center_quiet" ;;
             -v | --verbose) VERBOSE="true" ;;
             -V | --verbose-progress) VERBOSE_PROGRESS="true" && CURL_PROGRESS="" ;;
@@ -526,7 +538,7 @@ _process_arguments() {
             # Skip the sub folders and find recursively all the files and upload them.
             if [ -n "${SKIP_SUBDIRS}" ]; then
                 _print_center "justify" "Indexing files recursively.." "-"
-                FILENAMES="$(find "${input}" -type f)"
+                FILENAMES="$(_tmp='find "'${input}'" -type f -name "*" '${INCLUDE_FILES}' '${EXCLUDE_FILES}'' && eval "${_tmp}")"
 
                 if [ -n "${FILENAMES}" ]; then
                     NO_OF_FILES="$(($(printf "%s\n" "${FILENAMES}" | wc -l)))"
@@ -546,7 +558,7 @@ _process_arguments() {
             else
                 _print_center "justify" "$((NO_OF_SUB_FOLDERS)) Sub-folders found." "="
                 _print_center "justify" "Indexing files.." "="
-                FILENAMES="$(find "${input}" -type f)"
+                FILENAMES="$(_tmp='find "'${input}'" -type f -name "*" '${INCLUDE_FILES}' '${EXCLUDE_FILES}'' && eval "${_tmp}")"
 
                 if [ -n "${FILENAMES}" ]; then
                     NO_OF_FILES="$(($(printf "%s\n" "${FILENAMES}" | wc -l)))"