SAK-32218 Automatically import Sakai archives (sakaiproject#3948)

On startup if archives.import.source is set to a URL then download this file and each line is considered to contain a URL to a site archive. It will then download that archive and attempt to import it.

config/configuration/bundles/src/bundle/org/sakaiproject/config/bundle/default.sakai.properties

@@ -484,6 +484,11 @@
 # DEFAULT: Affiliate,Assistant,Instructor,Maintain,Organizer,Owner
 # archive.merge.filtered.roles={list of roles}
 
+# URL of text file containing URLs of Sakai archives to import on startup.
+# The file has one URL per line and lines starting with a comment are ignored.
+# DEFAULT:
+# archives.import.source=http://someserver.edu/file/import.txt
+
 # Upload limit per request, in MBs. 
 # "ceiling" is used for resources. "max" is used for attachments. 
 # In the case when both are set, resources are limited to the smaller of the two. If only one of the two are set, resource uploads

jobscheduler/scheduler-component-shared/pom.xml

@@ -45,6 +45,10 @@
             <groupId>org.sakaiproject.edu-services.course-management</groupId>
             <artifactId>coursemanagement-api</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.sakaiproject.common</groupId>
+            <artifactId>archive-api</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.sakaiproject.emailtemplateservice</groupId>
             <artifactId>emailtemplateservice-api</artifactId>
@@ -146,6 +150,7 @@
                 <includes>
                     <include>**/*.properties</include>
                     <include>**/*.xml</include>
+                    <include>**/*.zip</include>
                 </includes>
             </testResource>
         </testResources>

jobscheduler/scheduler-component-shared/src/java/org/sakaiproject/component/app/scheduler/AutoRun.java

@@ -58,7 +58,7 @@ public void setConfig(String config) {
     }
 
     public void init() {
-        if (serverConfigurationService.getBoolean(config, false)) {
+        if (config == null || serverConfigurationService.getBoolean(config, false)) {
             log.info("AutoRun running");
             Scheduler scheduler = schedulerManager.getScheduler();
 

jobscheduler/scheduler-component-shared/src/java/org/sakaiproject/component/app/scheduler/ConfigurableAutowiredJobBeanWrapper.java

@@ -0,0 +1,53 @@
+package org.sakaiproject.component.app.scheduler;
+
+import org.quartz.Job;
+import org.sakaiproject.api.app.scheduler.ConfigurableJobBeanWrapper;
+import org.sakaiproject.api.app.scheduler.ConfigurableJobProperty;
+import org.sakaiproject.api.app.scheduler.ConfigurableJobPropertyValidator;
+import org.sakaiproject.api.app.scheduler.DefaultJobPropertyValidator;
+
+import java.util.Set;
+
+/**
+ * Allow configurable autowired jobs.
+ */
+public class ConfigurableAutowiredJobBeanWrapper extends AutowiredJobBeanWrapper implements ConfigurableJobBeanWrapper {
+
+
+    private Set<ConfigurableJobProperty>
+            jobProperties;
+    private String
+            resourceBundleBase;
+    private ConfigurableJobPropertyValidator
+            validator = DEFAULT_VALIDATOR;
+    private static final DefaultJobPropertyValidator
+            DEFAULT_VALIDATOR = new DefaultJobPropertyValidator();
+
+    public ConfigurableAutowiredJobBeanWrapper(Class<? extends Job> aClass, String jobType) {
+        super(aClass, jobType);
+    }
+
+    public void setResourceBundleBase(String base) {
+        resourceBundleBase = base;
+    }
+
+    public String getResourceBundleBase() {
+        return resourceBundleBase;
+    }
+
+    public void setConfigurableJobProperties(Set<ConfigurableJobProperty> properties) {
+        jobProperties = properties;
+    }
+
+    public Set<ConfigurableJobProperty> getConfigurableJobProperties() {
+        return jobProperties;
+    }
+
+    public void setConfigurableJobPropertyValidator(ConfigurableJobPropertyValidator validator) {
+        this.validator = validator;
+    }
+
+    public ConfigurableJobPropertyValidator getConfigurableJobPropertyValidator() {
+        return validator;
+    }
+}

jobscheduler/scheduler-component-shared/src/java/org/sakaiproject/component/app/scheduler/jobs/autoimport/GetArchivesJob.java

@@ -0,0 +1,123 @@
+package org.sakaiproject.component.app.scheduler.jobs.autoimport;
+
+import org.apache.commons.io.filefilter.DirectoryFileFilter;
+import org.quartz.*;
+import org.sakaiproject.api.app.scheduler.SchedulerManager;
+import org.sakaiproject.component.api.ServerConfigurationService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import java.io.*;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.List;
+
+/**
+ * Attempts to download a file that lists all the site archives to import automatically.
+ */
+public class GetArchivesJob implements Job {
+
+    private final Logger log = LoggerFactory.getLogger(GetArchivesJob.class);
+
+    private ServerConfigurationService serverConfigurationService;
+
+    private SchedulerManager schedulerManager;
+
+    @Inject
+    public void setServerConfigurationService(ServerConfigurationService serverConfigurationService) {
+        this.serverConfigurationService = serverConfigurationService;
+    }
+
+    @Inject
+    public void setSchedulerManager(SchedulerManager schedulerManager) {
+        this.schedulerManager = schedulerManager;
+    }
+
+    @Override
+    public void execute(JobExecutionContext context) throws JobExecutionException {
+
+        String source = serverConfigurationService.getString("archives.import.source", null);
+        if (source == null) {
+            return;
+        }
+
+        log.info("Attempting to import archives listed in: "+ source);
+        try {
+            URL url = new URL(source);
+            URLConnection connection = url.openConnection();
+            connection.setRequestProperty("User-Agent", "Sakai Content Importer");
+            connection.setConnectTimeout(30000);
+            connection.setReadTimeout(30000);
+            // Now make the connection.
+            connection.connect();
+            try (InputStream inputStream = connection.getInputStream()) {
+                BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
+                String line;
+                while ((line = reader.readLine()) != null) {
+                    if (!line.isEmpty() && !line.startsWith("#")) {
+                        importArchive(line);
+                    }
+                }
+            }
+
+
+            String sakaiHome = serverConfigurationService.getSakaiHomePath();
+            String archiveHome = sakaiHome + "archive";
+
+            // Find all the folders and load them.
+            File archiveDirectory = new File(archiveHome);
+            File[] files = archiveDirectory.listFiles((FileFilter) DirectoryFileFilter.DIRECTORY);
+            if (files == null) {
+                return;
+            }
+            for (File dir: files) {
+                String dirName = dir.getName();
+                JobDataMap jobData = new JobDataMap();
+                jobData.put("folder", dirName);
+                JobDetail jobDetail = JobBuilder.newJob(ImportJob.class)
+                        .withIdentity("Import Job")
+                        .setJobData(jobData)
+                        .build();
+                Scheduler scheduler = schedulerManager.getScheduler();
+                try {
+                    scheduler.addJob(jobDetail, true, true);
+                    scheduler.triggerJob(jobDetail.getKey());
+                } catch (SchedulerException e) {
+                    log.warn("Problem adding job to scheduler to import "+ dirName, e);
+                }
+            }
+
+        } catch (IOException ioe) {
+            log.warn("Problem with "+ source + " "+ ioe.getMessage());
+        }
+    }
+
+    private void importArchive(String archive) {
+        String sakaiHome = serverConfigurationService.getSakaiHomePath();
+        String archiveHome = sakaiHome + "archive";
+
+        if (archive == null || archive.trim().length() == 0) {
+            log.warn("Empty archive setting.");
+            return;
+        }
+        log.info("Attempting to import: "+ archive);
+        try {
+            URL url = new URL(archive);
+            URLConnection connection = url.openConnection();
+            connection.setRequestProperty("User-Agent", "Sakai Content Importer");
+            connection.setConnectTimeout(30000);
+            connection.setReadTimeout(30000);
+            // Now make the connection.
+            connection.connect();
+            try (InputStream inputStream = connection.getInputStream()) {
+                List<ZipError> errors = ZipUtils.expandZip(inputStream, archiveHome);
+                for (ZipError error : errors) {
+                    log.info(error.toString());
+                }
+            }
+        } catch (IOException ioe) {
+            log.warn("Problem with "+ archive+ " "+ ioe.getMessage());
+        }
+    }
+}

jobscheduler/scheduler-component-shared/src/java/org/sakaiproject/component/app/scheduler/jobs/autoimport/ImportJob.java

@@ -0,0 +1,99 @@
+package org.sakaiproject.component.app.scheduler.jobs.autoimport;
+
+import org.quartz.Job;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.sakaiproject.archive.api.ArchiveService;
+import org.sakaiproject.component.api.ServerConfigurationService;
+import org.sakaiproject.site.api.SiteService;
+import org.sakaiproject.tool.api.Session;
+import org.sakaiproject.tool.api.SessionManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Configuration;
+
+import javax.inject.Inject;
+import java.io.File;
+
+/**
+ * This imports a folder into Sakai. The folder is stored inside the archive folder.
+ */
+public class ImportJob implements Job {
+
+    private final Logger log = LoggerFactory.getLogger(ImportJob.class);
+
+    private ServerConfigurationService serverConfigurationService;
+
+    private ArchiveService archiveService;
+
+    private SiteService siteService;
+
+    private SessionManager sessionManager;
+
+    private String folder;
+
+    @Inject
+    public void setServerConfigurationService(ServerConfigurationService serverConfigurationService) {
+        this.serverConfigurationService = serverConfigurationService;
+    }
+
+    @Inject
+    public void setArchiveService(ArchiveService archiveService) {
+        this.archiveService = archiveService;
+    }
+
+    @Inject
+    public void setSiteService(SiteService siteService) {
+        this.siteService = siteService;
+    }
+
+    @Inject
+    public void setSessionManager(SessionManager sessionManager) {
+        this.sessionManager = sessionManager;
+    }
+
+    public void setFolder(String folder) {
+        this.folder = folder;
+    }
+
+    @Override
+    public void execute(JobExecutionContext context) throws JobExecutionException {
+        String sakaiHome = serverConfigurationService.getSakaiHomePath();
+        String archiveHome = sakaiHome + "archive";
+        File archiveDirectory = new File(archiveHome);
+
+        File dir = new File(archiveDirectory, folder);
+        if (!dir.exists()) {
+            log.warn("Could not find {}, not importing.", dir);
+            return;
+        }
+        String siteId = trimArchive(folder);
+        if (siteService.siteExists(siteId)) {
+            log.info("Site already exists, not importing: " + siteId);
+            return;
+        }
+
+        log.info("Attempting to import: " + folder);
+        Session currentSession = sessionManager.getCurrentSession();
+        String oldId = currentSession.getUserId();
+        String oldEid = currentSession.getUserEid();
+        try {
+            currentSession.setUserId("admin");
+            currentSession.setUserEid("admin");
+            archiveService.merge(dir.getName(), siteId, null);
+        } catch (Exception e) {
+            log.warn("Failed to import " + dir.getAbsolutePath() + " to " + siteId + " " + e.getMessage());
+        } finally {
+            currentSession.setUserId(oldId);
+            currentSession.setUserEid(oldEid);
+        }
+
+    }
+
+    private String trimArchive(String siteId) {
+        if (siteId != null && siteId.endsWith("-archive")) {
+            siteId = siteId.substring(0, siteId.length() - "-archive".length());
+        }
+        return siteId;
+    }
+}

jobscheduler/scheduler-component-shared/src/java/org/sakaiproject/component/app/scheduler/jobs/autoimport/ZipError.java

@@ -0,0 +1,37 @@
+package org.sakaiproject.component.app.scheduler.jobs.autoimport;
+
+import java.io.File;
+
+/**
+ * An error that occured with the import.
+ * Easiest to just call {@see #toString()}.
+ * @author buckett
+ *
+ */
+public class ZipError {
+
+	private File file;
+	private String error;
+
+	public ZipError(File file, String error) {
+		this.file = file;
+		this.error = error;
+	}
+
+	/**
+	 * @return The file which had the problem, can be <code>null</code>.
+	 */
+	public File getFile() {
+		return file;
+	}
+
+	public String getError() {
+		return error;
+	}
+
+	public String toString() {
+		return (file == null)?
+				"Zipfile problem: "+ error:
+				"Error: "+ error+ " with "+ file.getAbsolutePath();
+	}
+}