Merge branch 'master' into master

1acboxer · Jan 15, 2018 · ecd7ff1 · ecd7ff1
2 parents c833fd2 + 8aad6dd
commit ecd7ff1
Show file tree

Hide file tree

Showing 2 changed files with 84 additions and 37 deletions.
diff --git a/README.md b/README.md
@@ -134,11 +134,15 @@ Show cURL progress meter
 * **-k**  
 Doesn't check for SSL certificates (insecure)
 
+* **-x &lt;FILENAME&gt;**  
+Ignores/excludes directories or files from syncing.
+-x filename -x directoryname. 
 
 **Examples:**
 ```bash
     ./dropbox_uploader.sh upload /etc/passwd /myfiles/passwd.old
     ./dropbox_uploader.sh upload *.zip /
+    ./dropbox_uploader.sh -x .git upload ./project /
     ./dropbox_uploader.sh download /backup.zip
     ./dropbox_uploader.sh delete /backup.zip
     ./dropbox_uploader.sh mkdir /myDir/
@@ -236,16 +240,18 @@ andrea@Dropbox:/$ ls
 andrea@DropBox:/ServerBackup$ get notes.txt
 ```
 
-## Building Docker image for Raspberry Pi
-You can build the docker image for a Raspberry Pi:
+## Running as Docker Container
+First build the docker image:
 ```bash
-pi@raspberrypi:/$ docker build https://github.com/sircuri/Dropbox-Uploader.git -f Dockerfile.pi -t <TAG>
+docker build https://github.com/sircuri/Dropbox-Uploader.git -f Dockerfile -t <TAG>
 ```
-
-## Running as Docker Container
-Run the container with:
+or for RaspBerry:
+```bash
+docker build https://github.com/sircuri/Dropbox-Uploader.git -f Dockerfile.pi -t <TAG>
+```
+then, you can run it as following:
 ```bash
-pi@raspberrypi:/$ docker run -i --rm --user=$(id -u):$(id -g) -v <LOCAL_CONFIG_PATH>:/config -v <YOUR_DATA_DIR_MOUNT> <TAG> <Arguments> 
+pi@raspberrypi:/$ docker run -i --rm --user=$(id -u):$(id -g) -v <LOCAL_CONFIG_PATH>:/config -v <YOUR_DATA_DIR_MOUNT>:/workdir <TAG> <Arguments> 
 ```
 This will store the auth token information in the given local directory in `<LOCAL_CONFIG_PATH>`. To ensure access to your mounted directories it can be important to pass a UID and GID to the docker deamon (as stated in the example by the --user argument)
 
@@ -259,5 +265,5 @@ To use a proxy, just set the mentioned environment variables via the docker `-e`
 ## Donations
 
  If you want to support this project, please consider donating:
- * PayPal: [email protected]
+ * PayPal: https://www.paypal.me/AndreaFabrizi83
  * BTC: 1JHCGAMpKqUwBjcT3Kno9Wd5z16K6WKPqG
diff --git a/dropbox_uploader.sh b/dropbox_uploader.sh
@@ -38,9 +38,9 @@ QUIET=0
 SHOW_PROGRESSBAR=0
 SKIP_EXISTING_FILES=0
 ERROR_STATUS=0
+EXCLUDE=()
 
 #Don't edit these...
-API_MIGRATE_V2="https://api.dropboxapi.com/1/oauth2/token_from_oauth1"
 API_LONGPOLL_FOLDER="https://notify.dropboxapi.com/2/files/list_folder/longpoll"
 API_CHUNKED_UPLOAD_START_URL="https://content.dropboxapi.com/2/files/upload_session/start"
 API_CHUNKED_UPLOAD_FINISH_URL="https://content.dropboxapi.com/2/files/upload_session/finish"
@@ -87,7 +87,7 @@ if [[ ! -d "$TMP_DIR" ]]; then
 fi
 
 #Look for optional config file parameter
-while getopts ":qpskdhf:" opt; do
+while getopts ":qpskdhfx:" opt; do
     case $opt in
 
     f)
@@ -118,6 +118,10 @@ while getopts ":qpskdhf:" opt; do
       HUMAN_READABLE_SIZE=1
     ;;
 
+    x)
+      EXCLUDE+=( $OPTARG )
+    ;;
+
     \?)
       echo "Invalid option: -$OPTARG" >&2
       exit 1
@@ -280,6 +284,7 @@ function usage
     echo -e "\t-h            Show file sizes in human readable format"
     echo -e "\t-p            Show cURL progress meter"
     echo -e "\t-k            Doesn't check for SSL certificates (insecure)"
+    echo -e "\t-x            Ignores/excludes directories or files from syncing. -x filename -x directoryname. example: -x .git"
 
     echo -en "\nFor more info and examples, please see the README file.\n\n"
     remove_temp_files
@@ -308,7 +313,7 @@ function check_http_response
         ;;
 
         #Missing CA certificates
-        60|58)
+        60|58|77)
             print "\nError: cURL is not able to performs peer SSL certificate verification.\n"
             print "Please, install the default ca-certificates bundle.\n"
             print "To do this in a Debian/Ubuntu based system, try:\n"
@@ -434,6 +439,14 @@ function db_upload
     local SRC=$(normalize_path "$1")
     local DST=$(normalize_path "$2")
 
+    for j in "${EXCLUDE[@]}"
+        do :
+            if [[ $(echo "$SRC" | grep "$j" | wc -l) -gt 0 ]]; then
+                print "Skipping excluded file/dir: "$j
+                return
+            fi
+    done
+
     #Checking if the file/dir exists
     if [[ ! -e $SRC && ! -d $SRC ]]; then
         print " > No such file or directory: $SRC\n"
@@ -576,20 +589,39 @@ function db_chunked_upload_file
     local FILE_SRC=$(normalize_path "$1")
     local FILE_DST=$(normalize_path "$2")
 
-    print " > Uploading \"$FILE_SRC\" to \"$FILE_DST\""
+
+    if [[ $SHOW_PROGRESSBAR == 1 && $QUIET == 0 ]]; then
+        VERBOSE=1
+        CURL_PARAMETERS="--progress-bar"        
+    else
+        VERBOSE=0
+        CURL_PARAMETERS="-L -s"        
+    fi
+
+
 
     local FILE_SIZE=$(file_size "$FILE_SRC")
     local OFFSET=0
     local UPLOAD_ID=""
     local UPLOAD_ERROR=0
     local CHUNK_PARAMS=""
 
+    ## Ceil division
+    let NUMBEROFCHUNK=($FILE_SIZE/1024/1024+$CHUNK_SIZE-1)/$CHUNK_SIZE
+
+    if [[ $VERBOSE == 1 ]]; then
+        print " > Uploading \"$FILE_SRC\" to \"$FILE_DST\" by $NUMBEROFCHUNK chunks ...\n"
+    else
+        print " > Uploading \"$FILE_SRC\" to \"$FILE_DST\" by $NUMBEROFCHUNK chunks "
+    fi    
+
     #Starting a new upload session
     $CURL_BIN $CURL_ACCEPT_CERTIFICATES -X POST -L -s --show-error --globoff -i -o "$RESPONSE_FILE" --header "Authorization: Bearer $OAUTH_ACCESS_TOKEN" --header "Dropbox-API-Arg: {\"close\": false}" --header "Content-Type: application/octet-stream" --data-binary @/dev/null "$API_CHUNKED_UPLOAD_START_URL" 2> /dev/null
     check_http_response
 
     SESSION_ID=$(sed -n 's/{"session_id": *"*\([^"]*\)"*.*/\1/p' "$RESPONSE_FILE")
 
+    chunkNumber=1
     #Uploading chunks...
     while ([[ $OFFSET != "$FILE_SIZE" ]]); do
 
@@ -599,19 +631,27 @@ function db_chunked_upload_file
         dd if="$FILE_SRC" of="$CHUNK_FILE" bs=1048576 skip=$OFFSET_MB count=$CHUNK_SIZE 2> /dev/null
         local CHUNK_REAL_SIZE=$(file_size "$CHUNK_FILE")
 
+        if [[ $VERBOSE == 1 ]]; then
+            print " >> Uploading chunk $chunkNumber of $NUMBEROFCHUNK\n"
+        fi
+
         #Uploading the chunk...
         echo > "$RESPONSE_FILE"
-        $CURL_BIN $CURL_ACCEPT_CERTIFICATES -X POST -L -s --show-error --globoff -i -o "$RESPONSE_FILE" --header "Authorization: Bearer $OAUTH_ACCESS_TOKEN" --header "Dropbox-API-Arg: {\"cursor\": {\"session_id\": \"$SESSION_ID\",\"offset\": $OFFSET},\"close\": false}" --header "Content-Type: application/octet-stream" --data-binary @"$CHUNK_FILE" "$API_CHUNKED_UPLOAD_APPEND_URL" 2> /dev/null
+        $CURL_BIN $CURL_ACCEPT_CERTIFICATES -X POST $CURL_PARAMETERS --show-error --globoff -i -o "$RESPONSE_FILE" --header "Authorization: Bearer $OAUTH_ACCESS_TOKEN" --header "Dropbox-API-Arg: {\"cursor\": {\"session_id\": \"$SESSION_ID\",\"offset\": $OFFSET},\"close\": false}" --header "Content-Type: application/octet-stream" --data-binary @"$CHUNK_FILE" "$API_CHUNKED_UPLOAD_APPEND_URL" 
         #check_http_response not needed, because we have to retry the request in case of error
 
-        let OFFSET=$OFFSET+$CHUNK_REAL_SIZE
-
         #Check
         if grep -q "^HTTP/1.1 200 OK" "$RESPONSE_FILE"; then
-            print "."
+            let OFFSET=$OFFSET+$CHUNK_REAL_SIZE
             UPLOAD_ERROR=0
+            if [[ $VERBOSE != 1 ]]; then
+                print "."
+            fi
+            ((chunkNumber=chunkNumber+1))
         else
-            print "*"
+            if [[ $VERBOSE != 1 ]]; then
+                print "*"
+            fi
             let UPLOAD_ERROR=$UPLOAD_ERROR+1
 
             #On error, the upload is retried for max 3 times
@@ -636,7 +676,6 @@ function db_chunked_upload_file
 
         #Check
         if grep -q "^HTTP/1.1 200 OK" "$RESPONSE_FILE"; then
-            print "."
             UPLOAD_ERROR=0
             break
         else
@@ -771,10 +810,10 @@ function db_download
 #$1 = Remote source file
 #$2 = Local destination file
 function db_download_file
-{
+{    
     local FILE_SRC=$(normalize_path "$1")
-    local FILE_DST=$(normalize_path "$2")
-
+    local FILE_DST=$(normalize_path "$2")   
+    
     if [[ $SHOW_PROGRESSBAR == 1 && $QUIET == 0 ]]; then
         CURL_PARAMETERS="-L --progress-bar"
         LINE_CR="\n"
@@ -789,6 +828,16 @@ function db_download_file
         return
     fi
 
+    # Checking if the file has the correct check sum
+    if [[ $TYPE != "ERR" ]]; then
+        sha_src=$(db_sha "$FILE_SRC")
+        sha_dst=$(db_sha_local "$FILE_DST")
+        if [[ $sha_src == $sha_dst && $sha_src != "ERR" ]]; then
+            print "> Skipping file \"$FILE_SRC\", file exists with the same hash\n"
+            return
+        fi
+    fi    
+
     #Creating the empty file, that for two reasons:
     #1) In this way I can check if the destination file is writable or not
     #2) Curl doesn't automatically creates files with 0 bytes size
@@ -1292,7 +1341,7 @@ function db_share
 function get_Share
 {
     local FILE_DST=$(normalize_path "$1")
-    $CURL_BIN $CURL_ACCEPT_CERTIFICATES -X POST -L -s --show-error --globoff -i -o "$RESPONSE_FILE" --header "Authorization: Bearer $OAUTH_ACCESS_TOKEN" --header "Content-Type: application/json" --data "{\"path\": \"$FILE_DST\"}" "$API_SHARE_LIST"
+    $CURL_BIN $CURL_ACCEPT_CERTIFICATES -X POST -L -s --show-error --globoff -i -o "$RESPONSE_FILE" --header "Authorization: Bearer $OAUTH_ACCESS_TOKEN" --header "Content-Type: application/json" --data "{\"path\": \"$FILE_DST\",\"direct_only\": true}" "$API_SHARE_LIST"
     check_http_response
 
     #Check
@@ -1437,15 +1486,16 @@ function db_sha_local
     fi
 
     while ([[ $OFFSET -lt "$FILE_SIZE" ]]); do
-        dd if="$FILE_SRC" of="$CHUNK_FILE" bs=4194304 skip=$SKIP count=1 2> /dev/null
+        dd if="$FILE" of="$CHUNK_FILE" bs=4194304 skip=$SKIP count=1 2> /dev/null
         local SHA=$(shasum -a 256 "$CHUNK_FILE" | awk '{print $1}')
         SHA_CONCAT="${SHA_CONCAT}${SHA}"
 
         let OFFSET=$OFFSET+4194304
         let SKIP=$SKIP+1
     done
 
-    echo $SHA_CONCAT | sed 's/\([0-9A-F]\{2\}\)/\\\\\\x\1/gI' | xargs printf | shasum -a 256 | awk '{print $1}'
+    shaHex=$(echo $SHA_CONCAT | sed 's/\([0-9A-F]\{2\}\)/\\x\1/gI')
+    echo -ne $shaHex | shasum -a 256 | awk '{print $1}'
 }
 
 ################
@@ -1462,19 +1512,10 @@ if [[ -e $CONFIG_FILE ]]; then
 
     #Checking if it's still a v1 API configuration file
     if [[ $APPKEY != "" || $APPSECRET != "" ]]; then
-        echo -ne "The config file contains the old v1 oauth tokens. A new oauth v2 token will be requested.\n"
-        echo -ne "Requesting new oauth2 token... "
-        $CURL_BIN $CURL_ACCEPT_CERTIFICATES -X POST -L -s --show-error --globoff -i -o "$RESPONSE_FILE" "$API_MIGRATE_V2/?oauth_consumer_key=$APPKEY&oauth_token=$OAUTH_ACCESS_TOKEN&oauth_signature_method=PLAINTEXT&oauth_signature=$APPSECRET%26$OAUTH_ACCESS_TOKEN_SECRET&oauth_timestamp=$(utime)&oauth_nonce=$RANDOM" 2> /dev/null
-        OAUTH_ACCESS_TOKEN=$(sed -n 's/.*access_token": "\([^"]*\).*/\1/p' "$RESPONSE_FILE")
-
-        if [[ $OAUTH_ACCESS_TOKEN == "" ]]; then
-            echo "Error getting access tocken, please try again!"
-            remove_temp_files
-            exit 1
-        fi
-
-        echo "DONE"
-        echo "OAUTH_ACCESS_TOKEN=$OAUTH_ACCESS_TOKEN" > "$CONFIG_FILE"
+        echo -ne "The config file contains the old deprecated v1 oauth tokens.\n"
+        echo -ne "Please run again the script and follow the configuration wizard. The old configuration file has been backed up to $CONFIG_FILE.old\n"
+        mv "$CONFIG_FILE" "$CONFIG_FILE".old
+        exit 1
     fi
 
     #Checking loaded data