Merge pull request github#298 from github/trackcomplete

Track completeness of Elasticsearch JSON dumps
Laurie790Ann · Apr 8, 2017 · 05eb5d2 · 05eb5d2
2 parents 342592c + 04f141d
commit 05eb5d2
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 5 deletions.
diff --git a/share/github-backup-utils/ghe-backup-es-audit-log b/share/github-backup-utils/ghe-backup-es-audit-log
@@ -34,11 +34,15 @@ fi
 current_index=audit_log-$(ghe-ssh "$host" 'date +"%Y-%m"')
 
 for index in $indices; do
-  if [ -f $GHE_DATA_DIR/current/audit-log/$index.gz -a $index \< $current_index ]; then
-    # Hard link any older indices since they are read only and won't change
+  if [ -f $GHE_DATA_DIR/current/audit-log/$index.gz -a -f $GHE_DATA_DIR/current/audit-log/$index.gz.complete -a $index \< $current_index ]; then
+    # Hard link any older indices that are complete, since these won't change
     ln $GHE_DATA_DIR/current/audit-log/$index.gz $GHE_SNAPSHOT_DIR/audit-log/$index.gz
+    ln $GHE_DATA_DIR/current/audit-log/$index.gz.complete $GHE_SNAPSHOT_DIR/audit-log/$index.gz.complete
   else
     ghe-ssh "$host" "/usr/local/share/enterprise/ghe-es-dump-json \"http://localhost:$es_port/$index\"" | gzip > $GHE_SNAPSHOT_DIR/audit-log/$index.gz
+    if [ $index \< $current_index ]; then
+      touch $GHE_SNAPSHOT_DIR/audit-log/$index.gz.complete
+    fi
   fi
 done
 

diff --git a/share/github-backup-utils/ghe-backup-es-hookshot b/share/github-backup-utils/ghe-backup-es-hookshot
@@ -21,14 +21,19 @@ ghe_remote_version_required "$host"
 mkdir -p "$GHE_SNAPSHOT_DIR/hookshot"
 
 indices=$(ghe-ssh "$host" 'curl -s "localhost:9201/_cat/indices/hookshot-logs-*"' | cut -d ' ' -f 3)
+
 current_index=hookshot-logs-$(ghe-ssh "$host" 'date +"%Y-%m-%d"')
 
 for index in $indices; do
-  if [ -f $GHE_DATA_DIR/current/hookshot/$index.gz -a $index \< $current_index ]; then
-    # Hard link any older indices since they are read only and won't change
+  if [ -f $GHE_DATA_DIR/current/hookshot/$index.gz -a -f $GHE_DATA_DIR/current/hookshot/$index.gz.complete -a $index \< $current_index ]; then
+    # Hard link any older indices that are complete, since these won't change
     ln $GHE_DATA_DIR/current/hookshot/$index.gz $GHE_SNAPSHOT_DIR/hookshot/$index.gz
+    ln $GHE_DATA_DIR/current/hookshot/$index.gz.complete $GHE_SNAPSHOT_DIR/hookshot/$index.gz.complete
   else
-    ghe-ssh "$host" "/usr/local/share/enterprise/ghe-es-dump-json 'http://localhost:9201/$index'" | gzip > $GHE_SNAPSHOT_DIR/hookshot/$index.gz
+    ghe-ssh "$host" "/usr/local/share/enterprise/ghe-es-dump-json \"http://localhost:9201/$index\"" | gzip > $GHE_SNAPSHOT_DIR/hookshot/$index.gz
+    if [ $index \< $current_index ]; then
+      touch $GHE_SNAPSHOT_DIR/hookshot/$index.gz.complete
+    fi
   fi
 done