Skip to content

Commit

Permalink
Merge pull request getodk#5025 from seadowg/media-files
Browse files Browse the repository at this point in the history 
Copy media files from old version rather than redownloading
  • Loading branch information
@grzesiek2010
grzesiek2010 authored Mar 3, 2022
2 parents 903639f + 11fb2d6 commit 8c63c96
Show file tree
Hide file tree
Showing 6 changed files with 321 additions and 180 deletions.
15 changes: 8 additions & 7 deletions ...oid/listeners/FormDownloaderListener.java → .../odk/collect/async/OngoingWorkListener.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.odk.collect.async

package org.odk.collect.android.listeners;

public interface FormDownloaderListener {

void progressUpdate(String currentFile, String progress, String total);

boolean isTaskCancelled();
/**
* Allows a client of some ongoing work to receive updates on progress and to provide a signal
* that the work should be cancelled.
*/
interface OngoingWorkListener {
fun progressUpdate(progress: Int)
val isCancelled: Boolean
}
59 changes: 59 additions & 0 deletions collect_app/src/main/java/org/odk/collect/android/formmanagement/FormMediaDownloader.kt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package org.odk.collect.android.formmanagement

import org.odk.collect.android.utilities.FileUtils.copyFile
import org.odk.collect.android.utilities.FileUtils.interuptablyWriteFile
import org.odk.collect.async.OngoingWorkListener
import org.odk.collect.forms.Form
import org.odk.collect.forms.FormSource
import org.odk.collect.forms.FormSourceException
import org.odk.collect.forms.FormsRepository
import org.odk.collect.forms.MediaFile
import org.odk.collect.shared.strings.Md5.getMd5Hash
import java.io.File
import java.io.IOException

class FormMediaDownloader(
private val formsRepository: FormsRepository,
private val formSource: FormSource
) {

@Throws(IOException::class, FormSourceException::class, InterruptedException::class)
fun download(
formToDownload: ServerFormDetails,
files: List<MediaFile>,
tempMediaPath: String,
tempDir: File,
stateListener: OngoingWorkListener
) {
val tempMediaDir = File(tempMediaPath).also { it.mkdir() }

files.forEachIndexed { i, mediaFile ->
stateListener.progressUpdate(i + 1)

val tempMediaFile = File(tempMediaDir, mediaFile.filename)

searchForExistingMediaFile(formToDownload, mediaFile).let {
if (it != null) {
copyFile(it, tempMediaFile)
} else {
val file = formSource.fetchMediaFile(mediaFile.downloadUrl)
interuptablyWriteFile(file, tempMediaFile, tempDir, stateListener)
}
}
}
}

private fun searchForExistingMediaFile(
formToDownload: ServerFormDetails,
mediaFile: MediaFile
): File? {
val allFormVersions = formsRepository.getAllByFormId(formToDownload.formId)
return allFormVersions.map { form: Form ->
File(form.formMediaPath, mediaFile.filename)
}.firstOrNull { file: File ->
val currentFileHash = getMd5Hash(file)
val downloadFileHash = mediaFile.hash
file.exists() && currentFileHash.contentEquals(downloadFileHash)
}
}
}
171 changes: 18 additions & 153 deletions collect_app/src/main/java/org/odk/collect/android/formmanagement/ServerFormDownloader.java
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,25 @@
package org.odk.collect.android.formmanagement;

import static org.apache.commons.io.FileUtils.deleteDirectory;
import static org.odk.collect.android.analytics.AnalyticsEvents.DOWNLOAD_SAME_FORMID_VERSION_DIFFERENT_HASH;
import static org.odk.collect.android.utilities.FileUtils.interuptablyWriteFile;

import org.jetbrains.annotations.NotNull;
import org.odk.collect.analytics.Analytics;
import org.odk.collect.android.R;
import org.odk.collect.android.analytics.AnalyticsUtils;
import org.odk.collect.android.application.Collect;
import org.odk.collect.android.listeners.FormDownloaderListener;
import org.odk.collect.android.utilities.FileUtils;
import org.odk.collect.android.utilities.FormNameUtils;
import org.odk.collect.async.OngoingWorkListener;
import org.odk.collect.forms.Form;
import org.odk.collect.forms.FormSource;
import org.odk.collect.forms.FormSourceException;
import org.odk.collect.forms.FormsRepository;
import org.odk.collect.forms.MediaFile;
import org.odk.collect.shared.strings.Md5;
import org.odk.collect.shared.strings.Validator;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
import java.util.Map;
import java.util.UUID;
Expand All @@ -30,9 +29,6 @@

import timber.log.Timber;

import static org.apache.commons.io.FileUtils.deleteDirectory;
import static org.odk.collect.android.analytics.AnalyticsEvents.DOWNLOAD_SAME_FORMID_VERSION_DIFFERENT_HASH;

public class ServerFormDownloader implements FormDownloader {

private final FormsRepository formsRepository;
Expand Down Expand Up @@ -77,7 +73,7 @@ public void downloadForm(ServerFormDetails form, @Nullable ProgressReporter prog
tempDir.mkdirs();

try {
FormDownloaderListener stateListener = new ProgressReporterAndSupplierStateListener(progressReporter, isCancelled);
OngoingWorkListener stateListener = new ProgressReporterAndSupplierStateListener(progressReporter, isCancelled);
processOneForm(form, stateListener, tempDir, formsDirPath, formMetadataParser);
} catch (FormSourceException e) {
throw new FormDownloadException.FormSourceError(e);
Expand All @@ -90,7 +86,7 @@ public void downloadForm(ServerFormDetails form, @Nullable ProgressReporter prog
}
}

private void processOneForm(ServerFormDetails fd, FormDownloaderListener stateListener, File tempDir, String formsDirPath, FormMetadataParser formMetadataParser) throws FormDownloadException, FormSourceException {
private void processOneForm(ServerFormDetails fd, OngoingWorkListener stateListener, File tempDir, String formsDirPath, FormMetadataParser formMetadataParser) throws FormDownloadException, FormSourceException {
// use a temporary media path until everything is ok.
String tempMediaPath = new File(tempDir, "media").getAbsolutePath();
FileResult fileResult = null;
Expand All @@ -102,17 +98,18 @@ private void processOneForm(ServerFormDetails fd, FormDownloaderListener stateLi

// download media files if there are any
if (fd.getManifest() != null && !fd.getManifest().getMediaFiles().isEmpty()) {
downloadMediaFiles(tempMediaPath, stateListener, fd.getManifest().getMediaFiles(), tempDir, fileResult.file.getName());
FormMediaDownloader mediaDownloader = new FormMediaDownloader(formsRepository, formSource);
mediaDownloader.download(fd, fd.getManifest().getMediaFiles(), tempMediaPath, tempDir, stateListener);
}
} catch (FormDownloadException.DownloadingInterrupted e) {
} catch (FormDownloadException.DownloadingInterrupted | InterruptedException e) {
Timber.i(e);
cleanUp(fileResult, tempMediaPath);
throw new FormDownloadException.DownloadingInterrupted();
} catch (IOException e) {
throw new FormDownloadException.DiskError();
}

if (stateListener != null && stateListener.isTaskCancelled()) {
if (stateListener != null && stateListener.isCancelled()) {
cleanUp(fileResult, tempMediaPath);
throw new FormDownloadException.DownloadingInterrupted();
}
Expand All @@ -132,7 +129,7 @@ private void processOneForm(ServerFormDetails fd, FormDownloaderListener stateLi
}
}

if (stateListener != null && stateListener.isTaskCancelled()) {
if (stateListener != null && stateListener.isCancelled()) {
throw new FormDownloadException.DownloadingInterrupted();
}

Expand Down Expand Up @@ -239,12 +236,12 @@ private Form saveNewForm(Map<String, String> formInfo, File formFile, String med
* Takes the formName and the URL and attempts to download the specified file. Returns a file
* object representing the downloaded file.
*/
private FileResult downloadXform(String formName, String url, FormDownloaderListener stateListener, File tempDir, String formsDirPath) throws FormSourceException, IOException, FormDownloadException.DownloadingInterrupted {
private FileResult downloadXform(String formName, String url, OngoingWorkListener stateListener, File tempDir, String formsDirPath) throws FormSourceException, IOException, FormDownloadException.DownloadingInterrupted, InterruptedException {
InputStream xform = formSource.fetchForm(url);

String fileName = getFormFileName(formName, formsDirPath);
File tempFormFile = new File(tempDir + File.separator + fileName);
writeFile(xform, tempFormFile, tempDir, stateListener);
interuptablyWriteFile(xform, tempFormFile, tempDir, stateListener);

// we've downloaded the file, and we may have renamed it
// make sure it's not the same as a file we already have
Expand All @@ -260,138 +257,6 @@ private FileResult downloadXform(String formName, String url, FormDownloaderList
}
}

/**
* Common routine to take a downloaded document save the contents in the file
* 'file'. Shared by media file download and form file download.
* <p>
* SurveyCTO: The file is saved into a temp folder and is moved to the final place if everything
* is okay, so that garbage is not left over on cancel.
*/
private void writeFile(InputStream inputStream, File destinationFile, File tempDir, FormDownloaderListener stateListener)
throws IOException, FormDownloadException.DownloadingInterrupted {

File tempFile = File.createTempFile(
destinationFile.getName(),
".tempDownload",
tempDir
);

// WiFi network connections can be renegotiated during a large form download sequence.
// This will cause intermittent download failures. Silently retry once after each
// failure. Only if there are two consecutive failures do we abort.
boolean success = false;
int attemptCount = 0;
final int maxAttemptCount = 2;
while (!success && ++attemptCount <= maxAttemptCount) {
// write connection to file
InputStream is = null;
OutputStream os = null;

try {
is = inputStream;
os = new FileOutputStream(tempFile);

byte[] buf = new byte[4096];
int len;
while ((len = is.read(buf)) > 0 && (stateListener == null || !stateListener.isTaskCancelled())) {
os.write(buf, 0, len);
}
os.flush();
success = true;

} catch (Exception e) {
Timber.e(e.toString());
// silently retry unless this is the last attempt,
// in which case we rethrow the exception.

FileUtils.deleteAndReport(tempFile);

if (attemptCount == maxAttemptCount) {
throw e;
}
} finally {
if (os != null) {
try {
os.close();
} catch (Exception e) {
Timber.e(e);
}
}
if (is != null) {
try {
// ensure stream is consumed...
final long count = 1024L;
while (is.skip(count) == count) {
// skipping to the end of the http entity
}
} catch (Exception e) {
// no-op
}
try {
is.close();
} catch (Exception e) {
Timber.w(e);
}
}
}

if (stateListener != null && stateListener.isTaskCancelled()) {
FileUtils.deleteAndReport(tempFile);
throw new FormDownloadException.DownloadingInterrupted();
}
}

Timber.d("Completed downloading of %s. It will be moved to the proper path...", tempFile.getAbsolutePath());

FileUtils.deleteAndReport(destinationFile);

String errorMessage = FileUtils.copyFile(tempFile, destinationFile);

if (destinationFile.exists()) {
Timber.d("Copied %s over %s", tempFile.getAbsolutePath(), destinationFile.getAbsolutePath());
FileUtils.deleteAndReport(tempFile);
} else {
String msg = Collect.getInstance().getString(R.string.fs_file_copy_error,
tempFile.getAbsolutePath(), destinationFile.getAbsolutePath(), errorMessage);
throw new RuntimeException(msg);
}
}

private void downloadMediaFiles(String tempMediaPath, FormDownloaderListener stateListener, List<MediaFile> files, File tempDir, String formFileName) throws IOException, FormDownloadException.DownloadingInterrupted, FormSourceException {
File tempMediaDir = new File(tempMediaPath);
tempMediaDir.mkdir();

for (int i = 0; i < files.size(); i++) {
if (stateListener != null) {
stateListener.progressUpdate("", String.valueOf(i + 1), "");
}

MediaFile toDownload = files.get(i);

File tempMediaFile = new File(tempMediaDir, toDownload.getFilename());
String finalMediaPath = FileUtils.constructMediaPath(formsDirPath + File.separator + formFileName);
File finalMediaFile = new File(finalMediaPath, toDownload.getFilename());

if (!finalMediaFile.exists()) {
InputStream mediaFile = formSource.fetchMediaFile(toDownload.getDownloadUrl());
writeFile(mediaFile, tempMediaFile, tempDir, stateListener);
} else {
String currentFileHash = Md5.getMd5Hash(finalMediaFile);
String downloadFileHash = validateHash(toDownload.getHash());

if (currentFileHash != null && downloadFileHash != null && !currentFileHash.contentEquals(downloadFileHash)) {
// if the hashes match, it's the same file otherwise replace it with the new one
InputStream mediaFile = formSource.fetchMediaFile(toDownload.getDownloadUrl());
writeFile(mediaFile, tempMediaFile, tempDir, stateListener);
} else {
// exists, and the hash is the same
// no need to download it again
Timber.i("Skipping media file fetch -- file hashes identical: %s", finalMediaFile.getAbsolutePath());
}
}
}
}

@NotNull
private static String getFormFileName(String formName, String formsDirPath) {
String formattedFormName = FormNameUtils.formatFilenameFromFormName(formName);
Expand Down Expand Up @@ -463,7 +328,7 @@ private boolean isNew() {
}
}

private static class ProgressReporterAndSupplierStateListener implements FormDownloaderListener {
private static class ProgressReporterAndSupplierStateListener implements OngoingWorkListener {
private final ProgressReporter progressReporter;
private final Supplier<Boolean> isCancelled;

Expand All @@ -473,14 +338,14 @@ private static class ProgressReporterAndSupplierStateListener implements FormDow
}

@Override
public void progressUpdate(String currentFile, String progress, String total) {
public void progressUpdate(int progress) {
if (progressReporter != null) {
progressReporter.onDownloadingMediaFile(Integer.parseInt(progress));
progressReporter.onDownloadingMediaFile(progress);
}
}

@Override
public boolean isTaskCancelled() {
public boolean isCancelled() {
if (isCancelled != null) {
return isCancelled.get();
} else {
Expand Down
Loading

0 comments on commit 8c63c96

Please sign in to comment.