Merged in milan/checksum-calculator (pull request #27)

Milan/checksum calculator

Approved-by: Hardik Modha <[email protected]>

.gitignore

@@ -3,6 +3,7 @@
 .project
 !.gitignore # except for .gitignore
 .settings
+/out
 
 *.tmproj
 

README.md

@@ -1,4 +1,4 @@
-# File-Uploader Plugin (Latest 3.0.9)
+# File-Uploader Plugin (Latest 3.1.0)
 
 Supported Grails 3.2.0
 
@@ -54,12 +54,21 @@ fileuploader {
             allowedExtensions = ["xls"]
             path = "./web-app/degree-applications"
             storageTypes = ""
+            //Explnation about this congiguration is given below
+            checksum {
+                calculate = true
+                algorithm = Algorithm.MD5                
+            }
         }
         userAvatar {
             maxSize = 1024 * 1024 * 2 //256 kbytes
             allowedExtensions = ["jpg","jpeg","gif","png"]
             storageTypes = "CDN"
             container = "anyContainerName"
+            checksum {
+                calculate = false
+                algorithm = Algorithm.SHA1                
+             }
         }
         logo {
             maxSize = 1024 * 1024 * 2 //256 kbytes
@@ -87,3 +96,26 @@ can be overwritten for group level configuration by setting **expirationPeriod**
    ```
    # Google Default Credentials
    export GOOGLE_APPLICATION_CREDENTIALS='/path/to/key.json'
+4. To Enable checksum checks and generation, define configuration as shown in the sample configuration, 
+```
+ groups {
+        degreeApplication {			// Non CDN files, will be stored in local directory.
+            maxSize = 1000 * 1024 //256 kbytes
+            allowedExtensions = ["xls"]
+            path = "./web-app/degree-applications"
+            storageTypes = ""
+            checksum {
+                calculate = true
+                algorithm = Algorithm.MD5                
+            }
+        }
+  }
+
+``` 
+If first flag is set to true, plugin will generate checksum for the uploaded file and try to find a file from database having same checksum. If any such file is found, then plugin will throw an **DuplicateFileException** exception.
+Second flag will tell plugin which algorithm to use to calculate the checksum. Currently, possible choices are,
+``` 
+Algorithm.MD5 and,
+Algorithm.SHA1
+```
+Second flag will be ignored if first flag is set to false. By default checksum calculation features is disabled. To enable provide valid configurations. 

build.gradle

@@ -18,12 +18,12 @@ buildscript {
         mavenCentral()
     }
     dependencies {
-        classpath "com.causecode.plugins:gradle-code-quality:1.0.0"
+        classpath "com.causecode.plugins:gradle-code-quality:1.0.1-RC1"
         classpath "org.grails:grails-gradle-plugin:$grailsVersion"
     }
 }
 
-version "3.0.9"
+version "3.1.0"
 group "com.causecode.plugins"
 
 apply plugin: "idea"

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Tue Nov 28 17:45:47 IST 2017
+#Tue Jan 30 19:18:03 IST 2018
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.4.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-3.4.1-all.zip

grails-app/domain/com/causecode/fileuploader/UFile.groovy

@@ -34,6 +34,12 @@ class UFile implements Serializable {
     String name
     String path
 
+    //Contains calculated hash value of file content
+    String checksum
+
+    //Algorithm Used to calculate Checksum
+    String checksumAlgorithm
+
     UFileType type
 
     Date dateCreated
@@ -48,12 +54,19 @@ class UFile implements Serializable {
         name blank: false
         fileGroup blank: false
         provider nullable: true
+        checksum nullable: true
+        checksumAlgorithm nullable: true
         dateCreated bindable: false
         lastUpdated bindable: false
     }
 
     static mapping = {
         path sqlType: 'text'
+
+        //Index on checksum
+        checksum index: true
+        //Index on checksumAlgorithm
+        checksumAlgorithm index: true
     }
 
     def afterDelete() {
@@ -118,6 +131,7 @@ enum UFileType {
     LOCAL(3)
 
     final int id
+
     UFileType(int id) {
         this.id = id
     }
@@ -137,6 +151,7 @@ enum CDNProvider {
     LOCAL(4)
 
     final int id
+
     CDNProvider(int id) {
         this.id = id
     }

grails-app/services/com/causecode/fileuploader/FileUploaderService.groovy

@@ -8,18 +8,21 @@
 package com.causecode.fileuploader
 
 import com.causecode.fileuploader.cdn.CDNFileUploader
+import com.causecode.fileuploader.cdn.amazon.AmazonCDNFileUploaderImpl
 import com.causecode.fileuploader.util.FileUploaderUtils
+import com.causecode.fileuploader.util.Time
+import com.causecode.fileuploader.util.checksum.ChecksumValidator
+import com.causecode.fileuploader.util.checksum.exceptions.DuplicateFileException
 import com.causecode.util.NucleusUtils
 import grails.core.GrailsApplication
 import grails.util.Holders
 import groovy.io.FileType
+import org.apache.commons.validator.UrlValidator
 import org.springframework.context.MessageSource
 import org.springframework.dao.DataIntegrityViolationException
 import org.springframework.web.multipart.MultipartFile
+
 import java.nio.channels.FileChannel
-import org.apache.commons.validator.UrlValidator
-import com.causecode.fileuploader.cdn.amazon.AmazonCDNFileUploaderImpl
-import com.causecode.fileuploader.util.Time
 
 /**
  * A service class for all fileUpload related operations.
@@ -72,15 +75,28 @@ class FileUploaderService {
      * @return
      */
     UFile saveFile(String group, def file, String customFileName = '', Object userInstance = null, Locale locale = null)
-            throws StorageConfigurationException, UploadFailureException, ProviderNotFoundException {
-
+            throws StorageConfigurationException, UploadFailureException,
+                    ProviderNotFoundException, FileNotFoundException,
+                    DuplicateFileException {
         Date expireOn
         long currentTimeMillis = System.currentTimeMillis()
         CDNProvider cdnProvider
         UFileType type = UFileType.LOCAL
         String path
-
         FileGroup fileGroupInstance = new FileGroup(group)
+        ChecksumValidator checksumValidator = new ChecksumValidator(fileGroupInstance)
+
+        if (checksumValidator.shouldCalculateChecksum()) {
+            UFile uFileInstance = UFile.findByChecksumAndChecksumAlgorithm(checksumValidator.getChecksum(file),
+                    checksumValidator.algorithm)
+            if (uFileInstance) {
+                throw new DuplicateFileException(
+                        "Checksum for file ${file.name} is ${checksumValidator.getChecksum(file)} and " +
+                                "that checksum refers to an existing file ${uFileInstance} on server"
+                )
+            }
+        }
+
         Map fileData = fileGroupInstance.getFileNameAndExtensions(file, customFileName)
 
         if (fileData.empty || !file) {
@@ -89,63 +105,61 @@ class FileUploaderService {
 
         fileGroupInstance.allowedExtensions(fileData, locale, group)
         fileGroupInstance.validateFileSize(fileData.fileSize, locale)
-
         // If group specific storage type is not defined then use the common storage type
         String storageTypes = fileGroupInstance.groupConfig.storageTypes ?: fileGroupInstance.config.storageTypes
 
         if (storageTypes == 'CDN') {
             type = UFileType.CDN_PUBLIC
-
             fileGroupInstance.scopeFileName(userInstance, fileData, group, currentTimeMillis)
             long expirationPeriod = getExpirationPeriod(group)
-
             File tempFile
 
             if (file instanceof File) {
-                /* No need to transfer a file of type File since its already in a temporary location.
-                * (Saves resource utilization)
-                */
+                // No need to transfer a file of type File since its already in a temporary location.
                 tempFile = file
             } else {
                 if (file instanceof MultipartFile) {
-                    tempFile = new File(newTemporaryDirectoryPath +
-                            "${fileData.fileName}.${fileData.fileExtension}")
-
+                    tempFile = getTempFilePathForMultipartFile(fileData.fileName, fileData.fileExtension)
                     file.transferTo(tempFile)
                 }
             }
 
             // Delete the temporary file when JVM exited since the base file is not required after upload
             tempFile.deleteOnExit()
-
             cdnProvider = fileGroupInstance.cdnProvider
 
             if (!cdnProvider) {
                 throw new StorageConfigurationException('Provider not defined in the Config. Please define one.')
             }
 
             expireOn = isPublicGroup(group) ? null : new Date(new Date().time + expirationPeriod * 1000)
-
             path = uploadFileToCloud(fileData, fileGroupInstance, tempFile)
-
         } else {
             path = fileGroupInstance.getLocalSystemPath(storageTypes, fileData, currentTimeMillis)
-
-            // Move file
             log.debug "Moving [$fileData.fileName] to [${path}]."
             moveFile(file, path)
         }
 
-        UFile ufile = new UFile([name: fileData.fileName, size: fileData.fileSize, path: path, type: type,
-                extension: fileData.fileExtension, expiresOn: expireOn, fileGroup: group, provider: cdnProvider])
-        NucleusUtils.save(ufile, true)
+        UFile ufile = new UFile(
+                [name     : fileData.fileName, size: fileData.fileSize, path: path, type: type,
+                 extension: fileData.fileExtension, expiresOn: expireOn, fileGroup: group, provider: cdnProvider])
+
+        if (checksumValidator.shouldCalculateChecksum()) {
+            ufile.checksum = checksumValidator.getChecksum(file)
+            ufile.checksumAlgorithm = checksumValidator.algorithm
+        }
 
+        NucleusUtils.save(ufile, true)
         return ufile
     }
 
+    private File getTempFilePathForMultipartFile(String fileName, String fileExtension) {
+        return new File(newTemporaryDirectoryPath + "${fileName}.${fileExtension}")
+    }
+
     /**
      * Method is used to upload file to cloud provider. Then it gets the path of uploaded file
-     * @params fileData, fileGroupInstance, tempFile
+     * @params fileData , fileGroupInstance, tempFile
      * @return path of uploaded file
      *
      */
@@ -176,7 +190,7 @@ class FileUploaderService {
 
     /**
      * Method is used to move file from temp directory to another.
-     * @params fileInstance, path
+     * @params fileInstance , path
      *
      */
     void moveFile(def file, String path) {

src/main/groovy/com/causecode/fileuploader/StorageException.groovy

@@ -3,7 +3,7 @@
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or
- * without modification, are not permitted.
+* without modification, are not permitted.
  */
 package com.causecode.fileuploader
 

src/main/groovy/com/causecode/fileuploader/util/checksum/Algorithm.groovy

@@ -0,0 +1,18 @@
+/*
+ * Copyright (c) 2011 - Present, CauseCode Technologies Pvt Ltd, India.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are not permitted.
+ */
+package com.causecode.fileuploader.util.checksum
+
+/**
+ * Enum to represent Algorithm to be used with hash calculation
+ * @author Milan Savaliya
+ * @since 3.1.0
+ */
+enum Algorithm {
+    MD5,
+    SHA1
+}

src/main/groovy/com/causecode/fileuploader/util/checksum/ChecksumConfig.groovy

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2011 - Present, CauseCode Technologies Pvt Ltd, India.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are not permitted.
+ */
+package com.causecode.fileuploader.util.checksum
+
+/**
+ * Checksum Config object to take decision weather to calculate checksum/hash or not, and if to calculate then with
+ * which algorithm
+ * @author Milan Savaliya
+ * @since 3.1.0
+ */
+
+class ChecksumConfig {
+    boolean calculate = false
+    Algorithm algorithm = Algorithm.MD5
+}