Skip to content

Commit

Permalink
New implementation of zipping batches of datafiles.
Browse files Browse the repository at this point in the history
  • Loading branch information
@landreev
landreev committed Feb 22, 2016
1 parent f37fdbd commit 41df7e5
Show file tree
Hide file tree
Showing 3 changed files with 96 additions and 171 deletions.
137 changes: 96 additions & 41 deletions src/main/java/edu/harvard/iq/dataverse/api/Access.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
import edu.harvard.iq.dataverse.authorization.users.GuestUser;
import edu.harvard.iq.dataverse.dataaccess.DataFileIO;
import edu.harvard.iq.dataverse.dataaccess.DataFileZipper;
import edu.harvard.iq.dataverse.dataaccess.FileAccessIO;
import edu.harvard.iq.dataverse.dataaccess.OptionalAccessService;
import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
Expand All @@ -40,6 +41,7 @@
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Properties;
import javax.inject.Inject;
Expand All @@ -60,6 +62,7 @@
import javax.ws.rs.ServiceUnavailableException;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.StreamingOutput;

/*
Custom API exceptions [NOT YET IMPLEMENTED]
Expand Down Expand Up @@ -399,61 +402,113 @@ public DownloadInstance tabularDatafileMetadataPreprocessed(@PathParam("fileId")
@Path("datafiles/{fileIds}")
@GET
@Produces({"application/zip"})
public ZippedDownloadInstance datafiles(@PathParam("fileIds") String fileIds, @QueryParam("key") String apiToken, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ {
ByteArrayOutputStream outStream = null;
public /*ZippedDownloadInstance*/ Response datafiles(@PathParam("fileIds") String fileIds, @QueryParam("key") String apiToken, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ {
// create a Download Instance without, without a primary Download Info object:
ZippedDownloadInstance downloadInstance = new ZippedDownloadInstance();
//ZippedDownloadInstance downloadInstance = new ZippedDownloadInstance();


long zipDownloadSizeLimit = systemConfig.getZipDownloadLimit();
if (zipDownloadSizeLimit > 0L) {
logger.fine("setting zip download size limit to " + zipDownloadSizeLimit + " bytes.");
downloadInstance.setSizeLimit(zipDownloadSizeLimit);


long setLimit = systemConfig.getZipDownloadLimit();
if (!(setLimit > 0L)) {
setLimit = DataFileZipper.DEFAULT_ZIPFILE_LIMIT;
}

long zipDownloadSizeLimit = setLimit;

logger.fine("setting zip download size limit to " + zipDownloadSizeLimit + " bytes.");

if (fileIds == null || fileIds.equals("")) {
throw new WebApplicationException(Response.Status.BAD_REQUEST);
}

String fileIdParams[] = fileIds.split(",");
if (fileIdParams != null && fileIdParams.length > 0) {
logger.fine(fileIdParams.length + " tokens;");
for (int i = 0; i < fileIdParams.length; i++) {
logger.fine("token: " + fileIdParams[i]);
Long fileId = null;
try {
fileId = new Long(fileIdParams[i]);
} catch (NumberFormatException nfe) {
fileId = null;
}
logger.fine("attempting to look up file id " + fileId);
DataFile file = dataFileService.find(fileId);
if (file != null) {
if (isAccessAuthorized(file, apiToken)) {
logger.fine("adding datafile (id=" + file.getId() + ") to the download list of the ZippedDownloadInstance.");
downloadInstance.addDataFile(file);
} else {
downloadInstance.setManifest(downloadInstance.getManifest() +
file.getFileMetadata().getLabel() + " IS RESTRICTED AND CANNOT BE DOWNLOADED\r\n");
}
StreamingOutput stream = new StreamingOutput() {

@Override
public void write(OutputStream os) throws IOException,
WebApplicationException {
String fileIdParams[] = fileIds.split(",");
DataFileZipper zipper = null;
boolean accessToUnrestrictedFileAuthorized = false;
String fileManifest = "";
long sizeTotal = 0L;

if (fileIdParams != null && fileIdParams.length > 0) {
logger.fine(fileIdParams.length + " tokens;");
for (int i = 0; i < fileIdParams.length; i++) {
logger.fine("token: " + fileIdParams[i]);
Long fileId = null;
try {
fileId = new Long(fileIdParams[i]);
} catch (NumberFormatException nfe) {
fileId = null;
}
if (fileId != null) {
logger.fine("attempting to look up file id " + fileId);
DataFile file = dataFileService.find(fileId);
if (file != null) {

if ((accessToUnrestrictedFileAuthorized && !file.isRestricted())
|| isAccessAuthorized(file, apiToken)) {

if (!file.isRestricted()) {
accessToUnrestrictedFileAuthorized = true;
}
logger.fine("adding datafile (id=" + file.getId() + ") to the download list of the ZippedDownloadInstance.");
//downloadInstance.addDataFile(file);

if (zipper == null) {
// This is the first file we can serve - so we now know that we are going to be able
// to produce some output.
zipper = new DataFileZipper(os);
zipper.setFileManifest(fileManifest);
response.setHeader("Content-disposition", "attachment; filename=\"dataverse_files.zip\"");
response.setHeader("Content-Type", "application/zip; name=\"dataverse_files.zip\"");
}
if (sizeTotal + file.getFilesize() < zipDownloadSizeLimit) {
sizeTotal += zipper.addFileToZipStream(file);
} else {
String fileName = file.getFileMetadata().getLabel();
String mimeType = file.getContentType();

zipper.addToManifest(fileName + " (" + mimeType + ") " + " skipped because the total size of the download bundle exceeded the limit of " + zipDownloadSizeLimit + " bytes.\r\n");
}
} else {
if (zipper == null) {
fileManifest = fileManifest + file.getFileMetadata().getLabel() + " IS RESTRICTED AND CANNOT BE DOWNLOADED\r\n";
} else {
zipper.addToManifest(file.getFileMetadata().getLabel() + " IS RESTRICTED AND CANNOT BE DOWNLOADED\r\n");
}
}

} else {
// Or should we just drop it and make a note in the Manifest?
throw new WebApplicationException(Response.Status.NOT_FOUND);
}
}
}
} else {
// Or should we just drop it and make a note in the Manifest?
throw new WebApplicationException(Response.Status.NOT_FOUND);
throw new WebApplicationException(Response.Status.BAD_REQUEST);
}
}
} else {
throw new WebApplicationException(Response.Status.BAD_REQUEST);
}

if (downloadInstance.getDataFiles().size() < 1) {
// This means the file ids supplied were valid, but none were
// accessible for this user:
throw new WebApplicationException(Response.Status.FORBIDDEN);
}

if (zipper == null) {
// If the DataFileZipper object is still NULL, it means that
// there were file ids supplied - but none of the corresponding
// files were accessible for this user.
// In which casew we don't bother generating any output, and
// just give them a 403:
throw new WebApplicationException(Response.Status.FORBIDDEN);
}

return downloadInstance;
// This will add the generated File Manifest to the zipped output,
// then flush and close the stream:
zipper.finalizeZipStream();

//os.flush();
//os.close();
}
};
return Response.ok(stream).build();
}

@Path("tempPreview/{fileSystemId}")
Expand Down
56 changes: 0 additions & 56 deletions src/main/java/edu/harvard/iq/dataverse/api/ZippedDownloadInstance.java
View file

This file was deleted.

74 changes: 0 additions & 74 deletions src/main/java/edu/harvard/iq/dataverse/api/ZippedDownloadInstanceWriter.java
View file

This file was deleted.

0 comments on commit 41df7e5

Please sign in to comment.