GEODE-4404: Move BackupWriter creation (apache#1521)

  * to facilitate future creation of backup plugins, the BackupWriter is created
    using generic information passed from gfsh

geode-core/src/main/java/org/apache/geode/admin/BackupStatus.java

@@ -14,6 +14,12 @@
  */
 package org.apache.geode.admin;
 
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.geode.cache.persistence.PersistentID;
+import org.apache.geode.distributed.DistributedMember;
+
 /**
  * The status of a backup operation, returned by
  * {@link AdminDistributedSystem#backupAllMembers(java.io.File,java.io.File)}.
@@ -23,5 +29,17 @@
  *             "{@docRoot}/org/apache/geode/management/package-summary.html">management</a></code>
  *             package instead
  */
-public interface BackupStatus extends org.apache.geode.management.BackupStatus {
+public interface BackupStatus {
+  /**
+   * Returns a map of disk stores that were successfully backed up. The key is an online distributed
+   * member. The value is the set of disk stores on that distributed member.
+   */
+  Map<DistributedMember, Set<PersistentID>> getBackedUpDiskStores();
+
+  /**
+   * Returns the set of disk stores that were known to be offline at the time of the backup. These
+   * members were not backed up. If this set is not empty the backup may not contain a complete
+   * snapshot of any partitioned regions in the distributed system.
+   */
+  Set<PersistentID> getOfflineDiskStores();
 }

geode-core/src/main/java/org/apache/geode/admin/internal/AdminDistributedSystemImpl.java

@@ -14,30 +14,80 @@
  */
 package org.apache.geode.admin.internal;
 
-import static org.apache.geode.distributed.ConfigurationProperties.*;
+import static org.apache.geode.distributed.ConfigurationProperties.DISABLE_TCP;
+import static org.apache.geode.distributed.ConfigurationProperties.LOCATORS;
+import static org.apache.geode.distributed.ConfigurationProperties.MCAST_ADDRESS;
+import static org.apache.geode.distributed.ConfigurationProperties.MCAST_PORT;
 
 import java.io.File;
 import java.io.IOException;
 import java.net.InetAddress;
-import java.util.*;
-import java.util.concurrent.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.StringTokenizer;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.FutureTask;
 
 import org.apache.logging.log4j.Logger;
 
 import org.apache.geode.CancelException;
 import org.apache.geode.SystemFailure;
-import org.apache.geode.admin.*;
+import org.apache.geode.admin.AdminException;
 import org.apache.geode.admin.Alert;
+import org.apache.geode.admin.AlertLevel;
 import org.apache.geode.admin.AlertListener;
+import org.apache.geode.admin.BackupStatus;
+import org.apache.geode.admin.CacheServer;
+import org.apache.geode.admin.CacheServerConfig;
+import org.apache.geode.admin.CacheVm;
+import org.apache.geode.admin.ConfigurationParameter;
+import org.apache.geode.admin.DistributedSystemConfig;
+import org.apache.geode.admin.DistributionLocator;
+import org.apache.geode.admin.DistributionLocatorConfig;
+import org.apache.geode.admin.GemFireHealth;
+import org.apache.geode.admin.ManagedEntity;
+import org.apache.geode.admin.ManagedEntityConfig;
+import org.apache.geode.admin.OperationCancelledException;
+import org.apache.geode.admin.RuntimeAdminException;
+import org.apache.geode.admin.SystemMember;
+import org.apache.geode.admin.SystemMemberCacheListener;
+import org.apache.geode.admin.SystemMembershipEvent;
+import org.apache.geode.admin.SystemMembershipListener;
 import org.apache.geode.cache.persistence.PersistentID;
 import org.apache.geode.distributed.DistributedMember;
 import org.apache.geode.distributed.FutureCancelledException;
-import org.apache.geode.distributed.internal.*;
+import org.apache.geode.distributed.internal.ClusterDistributionManager;
+import org.apache.geode.distributed.internal.DistributionManager;
+import org.apache.geode.distributed.internal.InternalDistributedSystem;
+import org.apache.geode.distributed.internal.InternalLocator;
 import org.apache.geode.distributed.internal.membership.InternalDistributedMember;
 import org.apache.geode.internal.Assert;
 import org.apache.geode.internal.Banner;
-import org.apache.geode.internal.admin.*;
-import org.apache.geode.internal.admin.remote.*;
+import org.apache.geode.internal.admin.ApplicationVM;
+import org.apache.geode.internal.admin.GemFireVM;
+import org.apache.geode.internal.admin.GfManagerAgent;
+import org.apache.geode.internal.admin.GfManagerAgentConfig;
+import org.apache.geode.internal.admin.GfManagerAgentFactory;
+import org.apache.geode.internal.admin.SSLConfig;
+import org.apache.geode.internal.admin.remote.CompactRequest;
+import org.apache.geode.internal.admin.remote.DistributionLocatorId;
+import org.apache.geode.internal.admin.remote.MissingPersistentIDsRequest;
+import org.apache.geode.internal.admin.remote.PrepareRevokePersistentIDRequest;
+import org.apache.geode.internal.admin.remote.RemoteApplicationVM;
+import org.apache.geode.internal.admin.remote.RemoteTransportConfig;
+import org.apache.geode.internal.admin.remote.RevokePersistentIDRequest;
+import org.apache.geode.internal.admin.remote.ShutdownAllRequest;
 import org.apache.geode.internal.cache.backup.BackupUtil;
 import org.apache.geode.internal.cache.persistence.PersistentMemberPattern;
 import org.apache.geode.internal.i18n.LocalizedStrings;
@@ -2313,7 +2363,7 @@ public BackupStatus backupAllMembers(File targetDir, File baselineDir) throws Ad
 
   public static BackupStatus backupAllMembers(DistributionManager dm, File targetDir,
       File baselineDir) throws AdminException {
-    return new BackupStatusImpl(BackupUtil.backupAllMembers(dm, targetDir, baselineDir));
+    return BackupUtil.backupAllMembers(dm, targetDir.toString(), baselineDir.toString());
   }
 
   public Map<DistributedMember, Set<PersistentID>> compactAllDiskStores() throws AdminException {

geode-core/src/main/java/org/apache/geode/internal/SystemAdmin.java

@@ -619,9 +619,8 @@ public static void backup(String targetDir) throws AdminException {
     InternalDistributedSystem ads = getAdminCnx();
 
     // Baseline directory should be null if it was not provided on the command line
-    BackupStatus status =
-        BackupUtil.backupAllMembers(ads.getDistributionManager(), new File(targetDir),
-            (SystemAdmin.baselineDir == null ? null : new File(SystemAdmin.baselineDir)));
+    BackupStatus status = BackupUtil.backupAllMembers(ads.getDistributionManager(), targetDir,
+        SystemAdmin.baselineDir);
 
     boolean incomplete = !status.getOfflineDiskStores().isEmpty();
 

geode-core/src/main/java/org/apache/geode/internal/cache/backup/AbstractBackupWriterConfig.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.internal.cache.backup;
+
+import java.util.Properties;
+
+import org.apache.commons.lang.StringUtils;
+
+abstract class AbstractBackupWriterConfig {
+
+  static final String TIMESTAMP = "TIMESTAMP";
+  static final String TYPE = "TYPE";
+
+  private final Properties properties;
+
+  AbstractBackupWriterConfig(Properties properties) {
+    this.properties = properties;
+  }
+
+  String getTimestamp() {
+    String value = properties.getProperty(TIMESTAMP);
+    if (StringUtils.isBlank(value)) {
+      throw new IllegalStateException("Timestamp is missing");
+    }
+    return value;
+  }
+
+  String getBackupType() {
+    String value = properties.getProperty(TYPE);
+    if (StringUtils.isBlank(value)) {
+      throw new IllegalStateException("Type is missing");
+    }
+    return value;
+  }
+
+  Properties getProperties() {
+    return properties;
+  }
+}

geode-core/src/main/java/org/apache/geode/internal/cache/backup/BackupDataStoreHelper.java

@@ -14,9 +14,9 @@
  */
 package org.apache.geode.internal.cache.backup;
 
-import java.io.File;
 import java.util.Collections;
 import java.util.Map;
+import java.util.Properties;
 import java.util.Set;
 
 import org.apache.geode.cache.persistence.PersistentID;
@@ -37,15 +37,15 @@ public class BackupDataStoreHelper {
 
   @SuppressWarnings("rawtypes")
   public static BackupDataStoreResult backupAllMembers(DistributionManager dm, Set recipients,
-      File targetDir, File baselineDir) {
+      Properties properties) {
     new FlushToDiskOperation(dm, dm.getId(), dm.getCache(), recipients, flushToDiskFactory).send();
 
     boolean abort = true;
     Map<DistributedMember, Set<PersistentID>> successfulMembers;
     Map<DistributedMember, Set<PersistentID>> existingDataStores;
     try {
       existingDataStores = new PrepareBackupOperation(dm, dm.getId(), dm.getCache(), recipients,
-          prepareBackupFactory, targetDir, baselineDir).send();
+          prepareBackupFactory, properties).send();
       abort = false;
     } finally {
       if (abort) {

geode-core/src/main/java/org/apache/geode/internal/cache/backup/BackupService.java

@@ -14,7 +14,6 @@
  */
 package org.apache.geode.internal.cache.backup;
 
-import java.io.File;
 import java.io.IOException;
 import java.util.HashSet;
 import java.util.List;
@@ -40,7 +39,7 @@
 import org.apache.geode.internal.logging.LoggingThreadGroup;
 
 public class BackupService {
-  Logger logger = LogService.getLogger();
+  private static final Logger logger = LogService.getLogger();
 
   public static final String DATA_STORES_TEMPORARY_DIRECTORY = "backupTemp_";
   private final ExecutorService executor;
@@ -71,10 +70,10 @@ public Thread newThread(final Runnable command) {
     return Executors.newSingleThreadExecutor(threadFactory);
   }
 
-  public HashSet<PersistentID> prepareBackup(InternalDistributedMember sender, File targetDir,
-      File baselineDir) throws IOException, InterruptedException {
+  public HashSet<PersistentID> prepareBackup(InternalDistributedMember sender, BackupWriter writer)
+      throws IOException, InterruptedException {
     validateRequestingAdmin(sender);
-    BackupTask backupTask = new BackupTask(cache, targetDir, baselineDir);
+    BackupTask backupTask = new BackupTask(cache, writer);
     if (!currentTask.compareAndSet(null, backupTask)) {
       throw new IOException("Another backup already in progress");
     }

geode-core/src/main/java/org/apache/geode/internal/cache/backup/BackupTask.java

@@ -14,9 +14,7 @@
  */
 package org.apache.geode.internal.cache.backup;
 
-import java.io.File;
 import java.io.IOException;
-import java.nio.file.Path;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -28,7 +26,6 @@
 import org.apache.geode.InternalGemFireError;
 import org.apache.geode.cache.DiskStore;
 import org.apache.geode.cache.persistence.PersistentID;
-import org.apache.geode.distributed.internal.membership.InternalDistributedMember;
 import org.apache.geode.internal.cache.DiskStoreImpl;
 import org.apache.geode.internal.cache.InternalCache;
 import org.apache.geode.internal.cache.Oplog;
@@ -44,38 +41,19 @@ public class BackupTask {
   private final RestoreScript restoreScript = new RestoreScript();
   private final InternalCache cache;
   private final CountDownLatch allowDestroys = new CountDownLatch(1);
-  private final String memberId;
   private final CountDownLatch locksAcquired = new CountDownLatch(1);
   private final CountDownLatch otherMembersReady = new CountDownLatch(1);
   private final HashSet<PersistentID> diskStoresWithData = new HashSet<>();
-  private final File targetDir;
-  private final File baselineDir;
   private final BackupWriter backupWriter;
 
   private volatile boolean isCancelled = false;
 
   private TemporaryBackupFiles temporaryFiles;
   private BackupFileCopier fileCopier;
 
-  BackupTask(InternalCache gemFireCache, File targetDir, File baselineDir) {
+  BackupTask(InternalCache gemFireCache, BackupWriter backupWriter) {
     this.cache = gemFireCache;
-    this.targetDir = targetDir;
-    this.baselineDir = baselineDir;
-    memberId = getCleanedMemberId();
-    backupWriter = createBackupWriter();
-  }
-
-  private BackupWriter createBackupWriter() {
-    BackupWriter writer;
-    Path backupDirectory = targetDir.toPath().resolve(memberId);
-    if (baselineDir == null) {
-      writer = new FileSystemBackupWriter(backupDirectory);
-    } else {
-      FileSystemIncrementalBackupLocation incrementalBaselineLocation =
-          new FileSystemIncrementalBackupLocation(baselineDir, memberId);
-      writer = new FileSystemBackupWriter(backupDirectory, incrementalBaselineLocation);
-    }
-    return writer;
+    this.backupWriter = backupWriter;
   }
 
   HashSet<PersistentID> getPreparedDiskStores() throws InterruptedException {
@@ -302,17 +280,6 @@ private DiskStoreBackup startDiskStoreBackup(DiskStoreImpl diskStore) throws IOE
     return backup;
   }
 
-  private String getCleanedMemberId() {
-    InternalDistributedMember memberId =
-        cache.getInternalDistributedSystem().getDistributedMember();
-    String vmId = memberId.toString();
-    return cleanSpecialCharacters(vmId);
-  }
-
-  private String cleanSpecialCharacters(String string) {
-    return string.replaceAll("[^\\w]+", "_");
-  }
-
   DiskStoreBackup getBackupForDiskStore(DiskStoreImpl diskStore) {
     return backupByDiskStore.get(diskStore);
   }

geode-core/src/main/java/org/apache/geode/internal/cache/backup/BackupUtil.java

@@ -14,14 +14,20 @@
  */
 package org.apache.geode.internal.cache.backup;
 
-import java.io.File;
+import static org.apache.geode.internal.cache.backup.AbstractBackupWriterConfig.TIMESTAMP;
+import static org.apache.geode.internal.cache.backup.AbstractBackupWriterConfig.TYPE;
+import static org.apache.geode.internal.cache.backup.FileSystemBackupWriterConfig.BASELINE_DIR;
+import static org.apache.geode.internal.cache.backup.FileSystemBackupWriterConfig.TARGET_DIR;
+
 import java.text.SimpleDateFormat;
 import java.util.Date;
+import java.util.Properties;
 import java.util.Set;
 
 import org.apache.geode.admin.internal.AdminDistributedSystemImpl;
 import org.apache.geode.cache.persistence.PersistentID;
 import org.apache.geode.distributed.internal.DistributionManager;
+import org.apache.geode.distributed.internal.membership.InternalDistributedMember;
 import org.apache.geode.internal.i18n.LocalizedStrings;
 import org.apache.geode.management.BackupStatus;
 import org.apache.geode.management.ManagementException;
@@ -33,19 +39,35 @@ private BackupUtil() {
     // do not instantiate
   }
 
-  public static BackupStatus backupAllMembers(DistributionManager dm, File targetDir,
-      File baselineDir) throws ManagementException {
-    BackupStatus status = null;
+  public static BackupStatus backupAllMembers(DistributionManager dm, String targetDirPath,
+      String baselineDirPath) {
+    Properties properties = createBackupProperties(targetDirPath, baselineDirPath);
+    return backupAllMembers(dm, properties);
+  }
+
+  static Properties createBackupProperties(String targetDirPath, String baselineDirPath) {
+    Properties properties = new Properties();
+    SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
+    properties.setProperty(TIMESTAMP, format.format(new Date()));
+    properties.setProperty(TYPE, "FileSystem");
+    properties.setProperty(TARGET_DIR, targetDirPath);
+    if (baselineDirPath != null) {
+      properties.setProperty(BASELINE_DIR, baselineDirPath);
+    }
+    return properties;
+  }
+
+  public static BackupStatus backupAllMembers(DistributionManager dm, Properties properties)
+      throws ManagementException {
+    BackupStatus status;
     if (BackupDataStoreHelper.obtainLock(dm)) {
       try {
         Set<PersistentID> missingMembers =
             AdminDistributedSystemImpl.getMissingPersistentMembers(dm);
-        Set recipients = dm.getOtherDistributionManagerIds();
+        Set<InternalDistributedMember> recipients = dm.getOtherDistributionManagerIds();
 
-        SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
-        targetDir = new File(targetDir, format.format(new Date()));
         BackupDataStoreResult result =
-            BackupDataStoreHelper.backupAllMembers(dm, recipients, targetDir, baselineDir);
+            BackupDataStoreHelper.backupAllMembers(dm, recipients, properties);
 
         // It's possible that when calling getMissingPersistentMembers, some members are
         // still creating/recovering regions, and at FinishBackupRequest.send, the