Allow simultaneous generation and sending

qwe15884889 · Dec 23, 2016 · b345ef6 · b345ef6
1 parent 02f65c1
commit b345ef6
Show file tree

Hide file tree

Showing 5 changed files with 112 additions and 86 deletions.
diff --git a/webdav-backup/README.md b/webdav-backup/README.md
@@ -7,6 +7,9 @@ You can also encrypt this backup and/or split this file into small pieces (as ce
 
 This backup can be instantaneous or scheduled, through CRON job (see `CRON` and `CRON_SCHEME` variables below).
 
+If the backup is split, the program will try to send a part as soon as it as been created and delete it as soon as it has been completed.
+Theoretically, if generation is as fast as sending, you should need only the space for two parts.
+
 For each WebDAV operation, you have a time out (defaults to 120 sec and editable).
 If the operation fails (whatever is the reason), it will be restarted until it succeed.
 

diff --git a/webdav-backup/docker/data/fixed-vars.sh b/webdav-backup/docker/data/fixed-vars.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+NOW=`date +%Y%m%d-%H%M%S`
+ARCHIVE_DIR=/var/archives
+ARCHIVE_NAME="backup-${NOW}.tar.gz"
+BACKUP_DIR="backup-${NOW}"
+BACKUP_DIST_DIR="${WEBDAV_HOST}/${DESTINATION_FOLDER}/${BACKUP_DIR}"
diff --git a/webdav-backup/docker/scripts/backup.sh b/webdav-backup/docker/scripts/backup.sh
@@ -2,16 +2,14 @@
 
 # Get others variables
 . /docker/data/vars.sh
-
-# Variables
-NOW=`date +%Y%m%d-%H%M%S`
-ARCHIVE_DIR=/var/archives
-ARCHIVE_NAME="backup-${NOW}.tar.gz"
-BACKUP_DIR="backup-${NOW}"
-BACKUP_DIST_DIR="${WEBDAV_HOST}/${DESTINATION_FOLDER}/${BACKUP_DIR}"
+. /docker/data/fixed-vars.sh
 
 echo "#--------------# Start Backup #--------------#"
 
+# Clean
+################################################################################
+rm -rf /var/archives/*
+
 # Prepare data
 ################################################################################
 
@@ -33,80 +31,6 @@ else
   tar -czf "${ARCHIVE_DIR}/${ARCHIVE_NAME}" /data
 fi
 
-ARCHIVE_DATA=`ls "${ARCHIVE_DIR}"`
-echo "Created files:"
-echo "${ARCHIVE_DATA}"
-
-# Send data
-################################################################################
-echo "### Transfert data to ${BACKUP_DIST_DIR}"
-
-# Create directories
-NEW_PATH=""
-directories=$(echo "${DESTINATION_FOLDER}/${BACKUP_DIR}" | tr "/" "\n")
-for dir in ${directories}; do
-  NEW_PATH="${NEW_PATH}${dir}/"
-  echo "Create ${NEW_PATH}"
-  SUCCESS=1
-	until [  ${SUCCESS} -eq 0 ]; do
-    curl -sS --user "${WEBDAV_USERNAME}:${WEBDAV_PASSWORD}" -m ${TIMEOUT} \
-         -X MKCOL "${WEBDAV_HOST}/${NEW_PATH}"
-    SUCCESS=$?
-  done
-done
-
-# Transfer data
-for file in ${ARCHIVE_DATA}; do
-  echo "Transfert ${file}"
-	SUCCESS=1
-	until [  ${SUCCESS} -eq 0 ]; do
-    curl -sS --user "${WEBDAV_USERNAME}:${WEBDAV_PASSWORD}" -m ${TIMEOUT} \
-         -T "${ARCHIVE_DIR}/${file}" "${BACKUP_DIST_DIR}/"
-		SUCCESS=$?
-	done
-done
-
-# Delete old backups
-################################################################################
-if [[ $BACKUP_NBR > 0 ]]; then
-  echo "### Delete oldest backups"
-
-  # Get list of files
-INFOS=`cadaver ${WEBDAV_HOST} <<EOF
-ls ${DESTINATION_FOLDER}
-EOF`
-
-  # Clean list of files
-  test='\bbackup\-[[:alpha:]]*\b'
-  files=(`echo $INFOS | tr '[[:space:]]' '\n' | grep "^$test"`)
-
-  # Get files to delete
-  i=0
-  NBR_FILES=${#files[@]}
-  FILES_TO_DELETE=()
-  for var in "${files[@]}"; do
-    if [[ $(( NBR_FILES - i )) > $BACKUP_NBR ]]; then
-      FILES_TO_DELETE+=($var)
-    fi
-    i=$((i+1))
-  done
-  echo "Files to delete : ${FILES_TO_DELETE[@]}"
-
-  # Delete oldest files
-  for var in "${FILES_TO_DELETE[@]}"; do
-    DELETED_DIR="${WEBDAV_HOST}/${DESTINATION_FOLDER}/${var}"
-    echo "Delete ${DELETED_DIR}"
-  	SUCCESS=1
-  	until [  ${SUCCESS} -eq 0 ]; do
-      curl -sS --user "${WEBDAV_USERNAME}:${WEBDAV_PASSWORD}" -m ${TIMEOUT} \
-           -X DELETE "${DELETED_DIR}"
-      SUCCESS=$?
-    done
-  done
-fi
-
-# Clean
-################################################################################
-rm -rf /var/archives/*
-
-echo "#--------------# End Backup #--------------#"
+# Create to indicate the end
+sleep 1 # Avoid same timestamp
+touch "${ARCHIVE_DIR}/end.txt"
diff --git a/webdav-backup/docker/scripts/entrypoint.sh b/webdav-backup/docker/scripts/entrypoint.sh
@@ -30,12 +30,14 @@ echo "TIMEOUT=${TIMEOUT}" >> /docker/data/vars.sh
 chmod 600 /docker/data/vars.sh
 
 # Create cron file
-echo "${CRON_SCHEME} root bash /docker/scripts/backup.sh >> /var/log/cron.log 2>&1" > /etc/cron.d/webdav-backup
+echo "${CRON_SCHEME} root bash /docker/scripts/backup.sh >> /var/log/cron.log 2>&1" >> /etc/cron.d/webdav-backup
+echo "${CRON_SCHEME} root sleep 3 && bash /docker/scripts/transfer.sh >> /var/log/cron.log 2>&1" >> /etc/cron.d/webdav-backup
 chmod 0644 /etc/cron.d/webdav-backup
 
 # Make backup
 if [ ${CRON} == false ]; then
-  bash /docker/scripts/backup.sh
+  bash /docker/scripts/backup.sh &
+  sleep 3 && bash /docker/scripts/transfer.sh
 else
   echo "Reading logs..."
   cron && tail -f /var/log/cron.log

diff --git a/webdav-backup/docker/scripts/transfer.sh b/webdav-backup/docker/scripts/transfer.sh
@@ -0,0 +1,90 @@
+#!/bin/bash
+
+# Get others variables
+. /docker/data/vars.sh
+. /docker/data/fixed-vars.sh
+
+# Send data
+################################################################################
+echo "### Transfert data to ${BACKUP_DIST_DIR}"
+
+# Create directories
+NEW_PATH=""
+directories=$(echo "${DESTINATION_FOLDER}/${BACKUP_DIR}" | tr "/" "\n")
+for dir in ${directories}; do
+  NEW_PATH="${NEW_PATH}${dir}/"
+  echo "Create ${NEW_PATH}"
+  SUCCESS=1
+	until [  ${SUCCESS} -eq 0 ]; do
+    curl -sS --user "${WEBDAV_USERNAME}:${WEBDAV_PASSWORD}" -m ${TIMEOUT} \
+         -X MKCOL "${WEBDAV_HOST}/${NEW_PATH}"
+    SUCCESS=$?
+  done
+done
+
+STOP=false
+until [ "${STOP}" == "true" ]; do
+  NBR_FILES=`ls -1 ${ARCHIVE_DIR} | wc -l`
+  file=`ls -t ${ARCHIVE_DIR} | tail -n 1`
+
+  if [ "${NBR_FILES}" -ge "2" ]; then
+    # Transfer data
+    echo "Transfert ${file}"
+  	SUCCESS=1
+  	until [  ${SUCCESS} -eq 0 ]; do
+      curl -sS --user "${WEBDAV_USERNAME}:${WEBDAV_PASSWORD}" -m ${TIMEOUT} \
+           -T "${ARCHIVE_DIR}/${file}" "${BACKUP_DIST_DIR}/"
+  		SUCCESS=$?
+  	done
+
+    # Delete file
+    rm -f "${ARCHIVE_DIR}/${file}"
+  elif [ "${file}" == "end.txt" ]; then
+    echo "End"
+    STOP=true
+  else
+    echo "Wait 5 seconds for next part..."
+    sleep 5
+  fi
+done
+
+# Delete old backups
+################################################################################
+if [[ $BACKUP_NBR > 0 ]]; then
+  echo "### Delete oldest backups"
+
+  # Get list of files
+INFOS=`cadaver ${WEBDAV_HOST} <<EOF
+ls ${DESTINATION_FOLDER}
+EOF`
+
+  # Clean list of files
+  test='\bbackup\-[[:alpha:]]*\b'
+  files=(`echo $INFOS | tr '[[:space:]]' '\n' | grep "^$test"`)
+
+  # Get files to delete
+  i=0
+  NBR_FILES=${#files[@]}
+  FILES_TO_DELETE=()
+  for var in "${files[@]}"; do
+    if [[ $(( NBR_FILES - i )) > $BACKUP_NBR ]]; then
+      FILES_TO_DELETE+=($var)
+    fi
+    i=$((i+1))
+  done
+  echo "Files to delete : ${FILES_TO_DELETE[@]}"
+
+  # Delete oldest files
+  for var in "${FILES_TO_DELETE[@]}"; do
+    DELETED_DIR="${WEBDAV_HOST}/${DESTINATION_FOLDER}/${var}"
+    echo "Delete ${DELETED_DIR}"
+  	SUCCESS=1
+  	until [  ${SUCCESS} -eq 0 ]; do
+      curl -sS --user "${WEBDAV_USERNAME}:${WEBDAV_PASSWORD}" -m ${TIMEOUT} \
+           -X DELETE "${DELETED_DIR}"
+      SUCCESS=$?
+    done
+  done
+fi
+
+echo "#--------------# End Backup #--------------#"