diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 5449d61825f..a03f51b7f76 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -22,6 +22,7 @@ import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.GuestUser; import edu.harvard.iq.dataverse.dataaccess.DataFileIO; +import edu.harvard.iq.dataverse.dataaccess.DataFileZipper; import edu.harvard.iq.dataverse.dataaccess.FileAccessIO; import edu.harvard.iq.dataverse.dataaccess.OptionalAccessService; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; @@ -40,6 +41,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.OutputStream; import java.util.ArrayList; import java.util.Properties; import javax.inject.Inject; @@ -60,6 +62,7 @@ import javax.ws.rs.ServiceUnavailableException; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response; +import javax.ws.rs.core.StreamingOutput; /* Custom API exceptions [NOT YET IMPLEMENTED] @@ -399,61 +402,113 @@ public DownloadInstance tabularDatafileMetadataPreprocessed(@PathParam("fileId") @Path("datafiles/{fileIds}") @GET @Produces({"application/zip"}) - public ZippedDownloadInstance datafiles(@PathParam("fileIds") String fileIds, @QueryParam("key") String apiToken, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { - ByteArrayOutputStream outStream = null; + public /*ZippedDownloadInstance*/ Response datafiles(@PathParam("fileIds") String fileIds, @QueryParam("key") String apiToken, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { // create a Download Instance without, without a primary Download Info object: - ZippedDownloadInstance downloadInstance = new ZippedDownloadInstance(); + //ZippedDownloadInstance downloadInstance = new ZippedDownloadInstance(); - long zipDownloadSizeLimit = systemConfig.getZipDownloadLimit(); - if (zipDownloadSizeLimit > 0L) { - logger.fine("setting zip download size limit to " + zipDownloadSizeLimit + " bytes."); - downloadInstance.setSizeLimit(zipDownloadSizeLimit); + + + long setLimit = systemConfig.getZipDownloadLimit(); + if (!(setLimit > 0L)) { + setLimit = DataFileZipper.DEFAULT_ZIPFILE_LIMIT; } + long zipDownloadSizeLimit = setLimit; + + logger.fine("setting zip download size limit to " + zipDownloadSizeLimit + " bytes."); + if (fileIds == null || fileIds.equals("")) { throw new WebApplicationException(Response.Status.BAD_REQUEST); } - String fileIdParams[] = fileIds.split(","); - if (fileIdParams != null && fileIdParams.length > 0) { - logger.fine(fileIdParams.length + " tokens;"); - for (int i = 0; i < fileIdParams.length; i++) { - logger.fine("token: " + fileIdParams[i]); - Long fileId = null; - try { - fileId = new Long(fileIdParams[i]); - } catch (NumberFormatException nfe) { - fileId = null; - } - logger.fine("attempting to look up file id " + fileId); - DataFile file = dataFileService.find(fileId); - if (file != null) { - if (isAccessAuthorized(file, apiToken)) { - logger.fine("adding datafile (id=" + file.getId() + ") to the download list of the ZippedDownloadInstance."); - downloadInstance.addDataFile(file); - } else { - downloadInstance.setManifest(downloadInstance.getManifest() + - file.getFileMetadata().getLabel() + " IS RESTRICTED AND CANNOT BE DOWNLOADED\r\n"); - } + StreamingOutput stream = new StreamingOutput() { + @Override + public void write(OutputStream os) throws IOException, + WebApplicationException { + String fileIdParams[] = fileIds.split(","); + DataFileZipper zipper = null; + boolean accessToUnrestrictedFileAuthorized = false; + String fileManifest = ""; + long sizeTotal = 0L; + + if (fileIdParams != null && fileIdParams.length > 0) { + logger.fine(fileIdParams.length + " tokens;"); + for (int i = 0; i < fileIdParams.length; i++) { + logger.fine("token: " + fileIdParams[i]); + Long fileId = null; + try { + fileId = new Long(fileIdParams[i]); + } catch (NumberFormatException nfe) { + fileId = null; + } + if (fileId != null) { + logger.fine("attempting to look up file id " + fileId); + DataFile file = dataFileService.find(fileId); + if (file != null) { + + if ((accessToUnrestrictedFileAuthorized && !file.isRestricted()) + || isAccessAuthorized(file, apiToken)) { + + if (!file.isRestricted()) { + accessToUnrestrictedFileAuthorized = true; + } + logger.fine("adding datafile (id=" + file.getId() + ") to the download list of the ZippedDownloadInstance."); + //downloadInstance.addDataFile(file); + + if (zipper == null) { + // This is the first file we can serve - so we now know that we are going to be able + // to produce some output. + zipper = new DataFileZipper(os); + zipper.setFileManifest(fileManifest); + response.setHeader("Content-disposition", "attachment; filename=\"dataverse_files.zip\""); + response.setHeader("Content-Type", "application/zip; name=\"dataverse_files.zip\""); + } + if (sizeTotal + file.getFilesize() < zipDownloadSizeLimit) { + sizeTotal += zipper.addFileToZipStream(file); + } else { + String fileName = file.getFileMetadata().getLabel(); + String mimeType = file.getContentType(); + + zipper.addToManifest(fileName + " (" + mimeType + ") " + " skipped because the total size of the download bundle exceeded the limit of " + zipDownloadSizeLimit + " bytes.\r\n"); + } + } else { + if (zipper == null) { + fileManifest = fileManifest + file.getFileMetadata().getLabel() + " IS RESTRICTED AND CANNOT BE DOWNLOADED\r\n"; + } else { + zipper.addToManifest(file.getFileMetadata().getLabel() + " IS RESTRICTED AND CANNOT BE DOWNLOADED\r\n"); + } + } + + } else { + // Or should we just drop it and make a note in the Manifest? + throw new WebApplicationException(Response.Status.NOT_FOUND); + } + } + } } else { - // Or should we just drop it and make a note in the Manifest? - throw new WebApplicationException(Response.Status.NOT_FOUND); + throw new WebApplicationException(Response.Status.BAD_REQUEST); } - } - } else { - throw new WebApplicationException(Response.Status.BAD_REQUEST); - } - if (downloadInstance.getDataFiles().size() < 1) { - // This means the file ids supplied were valid, but none were - // accessible for this user: - throw new WebApplicationException(Response.Status.FORBIDDEN); - } - + if (zipper == null) { + // If the DataFileZipper object is still NULL, it means that + // there were file ids supplied - but none of the corresponding + // files were accessible for this user. + // In which casew we don't bother generating any output, and + // just give them a 403: + throw new WebApplicationException(Response.Status.FORBIDDEN); + } - return downloadInstance; + // This will add the generated File Manifest to the zipped output, + // then flush and close the stream: + zipper.finalizeZipStream(); + + //os.flush(); + //os.close(); + } + }; + return Response.ok(stream).build(); } @Path("tempPreview/{fileSystemId}") diff --git a/src/main/java/edu/harvard/iq/dataverse/api/ZippedDownloadInstance.java b/src/main/java/edu/harvard/iq/dataverse/api/ZippedDownloadInstance.java deleted file mode 100644 index 887c66a449b..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/api/ZippedDownloadInstance.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ - -package edu.harvard.iq.dataverse.api; - -import edu.harvard.iq.dataverse.DataFile; -import java.util.ArrayList; -import java.util.List; - -/** - * - * @author Leonid Andreev - */ -public class ZippedDownloadInstance { - - private String Manifest = ""; - private long sizeLimit = -1; - - private List dataFiles = null; - - public ZippedDownloadInstance() { - dataFiles = new ArrayList<>(); - } - - public List getDataFiles() { - return dataFiles; - } - - public void setDataFiles(List dataFiles) { - this.dataFiles = dataFiles; - } - - public void addDataFile(DataFile dataFile) { - dataFiles.add(dataFile); - } - - public String getManifest() { - return Manifest; - } - - public void setManifest(String Manifest) { - this.Manifest = Manifest; - } - - public long getSizeLimit() { - return sizeLimit; - } - - public void setSizeLimit(long sizeLimit) { - this.sizeLimit = sizeLimit; - } - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/api/ZippedDownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/ZippedDownloadInstanceWriter.java deleted file mode 100644 index e94031cc10d..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/api/ZippedDownloadInstanceWriter.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ - -package edu.harvard.iq.dataverse.api; - -import java.lang.reflect.Type; -import java.lang.annotation.Annotation; -import java.io.OutputStream; -import java.io.IOException; - -import javax.ws.rs.WebApplicationException; - -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.MultivaluedMap; -import javax.ws.rs.core.Response; - -import javax.ws.rs.ext.MessageBodyWriter; -import javax.ws.rs.ext.Provider; - -import edu.harvard.iq.dataverse.dataaccess.*; -import java.util.logging.Logger; - -/** - * - * @author Leonid Andreev - */ -@Provider -public class ZippedDownloadInstanceWriter implements MessageBodyWriter { - - private static final Logger logger = Logger.getLogger(ZippedDownloadInstanceWriter.class.getCanonicalName()); - - - @Override - public boolean isWriteable(Class clazz, Type type, Annotation[] annotation, MediaType mediaType) { - return clazz == ZippedDownloadInstance.class; - } - - @Override - public long getSize(ZippedDownloadInstance di, Class clazz, Type type, Annotation[] annotation, MediaType mediaType) { - return -1; - } - - @Override - public void writeTo(ZippedDownloadInstance di, Class clazz, Type type, Annotation[] annotation, MediaType mediaType, MultivaluedMap httpHeaders, OutputStream outstream) throws IOException, WebApplicationException { - - if (di.getDataFiles() != null && di.getDataFiles().size() > 0) { - - httpHeaders.add("Content-disposition", "attachment; filename=\"dataverse_files.zip\""); - httpHeaders.add("Content-Type", "application/zip; name=\"dataverse_files.zip\""); - - DataFileZipper zipper = new DataFileZipper(); - - try { - if (di.getSizeLimit() > 0) { - zipper.zipFiles(di.getDataFiles(), outstream, di.getManifest(), di.getSizeLimit()); - } else { - zipper.zipFiles(di.getDataFiles(), outstream, di.getManifest()); - } - } catch (IOException ioe) { - throw new WebApplicationException(Response.Status.INTERNAL_SERVER_ERROR); - } - - return; - } - - logger.warning("empty list of extra arguments."); - throw new WebApplicationException(Response.Status.BAD_REQUEST); - - } - -}