Add support for file chunk upload

bz-dev · Nov 2, 2020 · 40164ee · 40164ee
1 parent f53d472
commit 40164ee
Show file tree

Hide file tree

Showing 3 changed files with 55 additions and 5 deletions.
diff --git a/src/main/kotlin/uk/ac/le/ember/labpipe/server/controllers/DatabaseController.kt b/src/main/kotlin/uk/ac/le/ember/labpipe/server/controllers/DatabaseController.kt
@@ -200,6 +200,12 @@ class DatabaseController {
                     MONGO.COLLECTIONS.API_ACCESS_ROLES.insertOne(record)
                 }
             }
+            runBlocking {
+                val record = ApiAccessRole(url = API.UPLOAD.FORM_FILE_CHUNK, roles = mutableSetOf(DEFAULT_TOKEN_ROLE.identifier, DEFAULT_OPERATOR_ROLE.identifier, DEFAULT_ADMIN_ROLE.identifier))
+                if (MONGO.COLLECTIONS.API_ACCESS_ROLES.findOne(ApiAccessRole::url eq record.url) == null) {
+                    MONGO.COLLECTIONS.API_ACCESS_ROLES.insertOne(record)
+                }
+            }
         }
     }
 }
diff --git a/src/main/kotlin/uk/ac/le/ember/labpipe/server/routes/RouteController.kt b/src/main/kotlin/uk/ac/le/ember/labpipe/server/routes/RouteController.kt
@@ -28,6 +28,7 @@ import uk.ac.le.ember.labpipe.server.services.listRecords
 import uk.ac.le.ember.labpipe.server.services.listStudies
 import uk.ac.le.ember.labpipe.server.services.saveRecord
 import uk.ac.le.ember.labpipe.server.services.uploadFile
+import uk.ac.le.ember.labpipe.server.services.uploadFileChunk
 import uk.ac.le.ember.labpipe.server.sessions.Runtime
 import java.time.LocalDateTime
 import java.util.concurrent.TimeUnit
@@ -277,6 +278,10 @@ class RouteController {
                 getRateLimiter(ctx = ctx, url = API.UPLOAD.FORM_FILE)
                 uploadFile(ctx) },
                 SecurityUtil.roles(AuthManager.ApiRole.AUTHORISED, AuthManager.ApiRole.TOKEN_AUTHORISED))
+            Runtime.server.post(API.UPLOAD.FORM_FILE, { ctx ->
+                getRateLimiter(ctx = ctx, url = API.UPLOAD.FORM_FILE_CHUNK)
+                uploadFileChunk(ctx) },
+                SecurityUtil.roles(AuthManager.ApiRole.AUTHORISED, AuthManager.ApiRole.TOKEN_AUTHORISED))
         }
     }
 }
diff --git a/src/main/kotlin/uk/ac/le/ember/labpipe/server/services/UploadService.kt b/src/main/kotlin/uk/ac/le/ember/labpipe/server/services/UploadService.kt
@@ -1,19 +1,27 @@
 package uk.ac.le.ember.labpipe.server.services
 
-import io.javalin.core.security.SecurityUtil
 import io.javalin.core.util.FileUtil
 import io.javalin.http.Context
-import uk.ac.le.ember.labpipe.server.*
+import org.apache.commons.codec.digest.DigestUtils.md5Hex
+import org.litote.kmongo.and
+import org.litote.kmongo.eq
+import org.litote.kmongo.findOne
+import uk.ac.le.ember.labpipe.server.FormFileChunkUpload
+import uk.ac.le.ember.labpipe.server.FormFileUpload
+import uk.ac.le.ember.labpipe.server.MONGO
+import uk.ac.le.ember.labpipe.server.Message
+import uk.ac.le.ember.labpipe.server.controllers.ConfigController.Companion.LabPipeConfig.Storage
 import uk.ac.le.ember.labpipe.server.sessions.Runtime
+import java.nio.file.Files
 import java.nio.file.Paths
 
-private fun uploadFile(ctx: Context): Context {
+fun uploadFile(ctx: Context): Context {
     val identifier = ctx.queryParam("identifier")
     identifier?.run {
-        var f = FormFileUpload(identifier = identifier)
+        val f = FormFileUpload(identifier = identifier)
         ctx.uploadedFiles("files").forEachIndexed { index, uploadedFile ->
             val newPath = Paths.get(
-                Runtime.lpConfig.uploadedPath,
+                Runtime.config[Storage.upload],
                 "${identifier}@${index}${uploadedFile.extension}"
             )
             FileUtil.streamToFile(
@@ -27,4 +35,35 @@ private fun uploadFile(ctx: Context): Context {
     return ctx.status(400).json(Message("Form action identifier is required."))
 }
 
+fun uploadFileChunk(ctx: Context): Context {
+    val identifier = ctx.queryParam("identifier")
+    val chunk = ctx.queryParam<Int>("chunk").get()
+    val total = ctx.queryParam<Int>("total").get()
+    val md5Chunk = ctx.formParam<String>("md5Chunk").get()
+    val md5Total = ctx.formParam<String>("md5Total").get()
+    val ext = ctx.queryParam<String>("ext").get()
+    identifier?.run {
+        val f = FormFileChunkUpload(identifier = identifier, chunk = chunk, total = total, md5Chunk = md5Chunk, md5Total = md5Total, ext = ext)
+        if (MONGO.COLLECTIONS.CHUNKED.findOne(and(FormFileChunkUpload::identifier eq identifier, FormFileChunkUpload::chunk eq chunk, FormFileChunkUpload::md5Total eq md5Total)) != null) {
+            return ctx.status(400).json(Message("File chunk already exists on server."))
+        }
+        if (ctx.uploadedFiles("files").size != 1) {
+            return ctx.status(400).json(Message("File chunk upload must have only one file uploaded each time."))
+        }
+        val newPath = Paths.get(
+            Runtime.config[Storage.upload],
+            "${identifier}@${total}-${chunk}.bin"
+        )
+        FileUtil.streamToFile(
+            ctx.uploadedFiles("files")[0].content, newPath.toString()
+        )
+        val inputStream = Files.newInputStream(newPath)
+        if (md5Hex(inputStream) != md5Chunk) {
+            return ctx.status(400).json(Message("File md5 does not match."))
+        }
+        f.file = newPath.toString()
+        MONGO.COLLECTIONS.CHUNKED.insertOne(f)
+        return ctx.status(200).json(Message("Uploaded completed."))
+    }
+    return ctx.status(400).json(Message("Form action identifier is required."))
 }