Thumbnail handling, cleanup

* Thumbnails can now be created with help of ImageMagick, which must be available on the system to work.
* Implemented systems for concurrent thumbnail request and processing for a large amount of files.

cache/captions/de_de

@@ -190,6 +190,7 @@
 			"logLevelCache":"Cache",
 			"logLevelCluster":"Cluster",
 			"logLevelCsv":"CSV-Import/Export",
+			"logLevelImager":"Bildverarbeitung",
 			"logLevelLdap":"LDAP",
 			"logLevelMail":"Mail",
 			"logLevelScheduler":"Aufgabenplaner",

cache/captions/en_us

@@ -190,6 +190,7 @@
 			"logLevelCache":"Cache",
 			"logLevelCluster":"Cluster",
 			"logLevelCsv":"CSV import/export",
+			"logLevelImager":"Image processing",
 			"logLevelLdap":"LDAP",
 			"logLevelMail":"Mail",
 			"logLevelScheduler":"Scheduler",

cache/captions/it_it

@@ -190,6 +190,7 @@
 			"logLevelCache":"Cache",
 			"logLevelCluster":"Cluster",
 			"logLevelCsv":"Importa/esporta CSV",
+			"logLevelImager":"Image processing",
 			"logLevelLdap":"LDAP",
 			"logLevelMail":"Posta",
 			"logLevelScheduler":"Pianificatore",

cache/captions/ro_ro

@@ -190,6 +190,7 @@
 			"logLevelCache":"Tampon",
 			"logLevelCluster":"Cluster",
 			"logLevelCsv":"CSV import/export",
+			"logLevelImager":"Image processing",
 			"logLevelLdap":"LDAP",
 			"logLevelMail":"Mail",
 			"logLevelScheduler":"Programare",

config/config.go

@@ -84,6 +84,7 @@ func SetLogLevels() {
 	log.SetLogLevel("cache", int(GetUint64("logCache")))
 	log.SetLogLevel("cluster", int(GetUint64("logCluster")))
 	log.SetLogLevel("csv", int(GetUint64("logCsv")))
+	log.SetLogLevel("imager", int(GetUint64("logImager")))
 	log.SetLogLevel("ldap", int(GetUint64("logLdap")))
 	log.SetLogLevel("mail", int(GetUint64("logMail")))
 	log.SetLogLevel("scheduler", int(GetUint64("logScheduler")))

config/config_store.go

@@ -27,11 +27,12 @@ var (
 		"clusterNodeMissingAfter", "dbTimeoutCsv", "dbTimeoutDataRest",
 		"dbTimeoutDataWs", "dbTimeoutIcs", "filesKeepDaysDeleted",
 		"filesKeepDaysUnassigned", "icsDaysPost", "icsDaysPre", "icsDownload",
-		"logApplication", "logBackup", "logCache", "logCluster", "logCsv",
-		"logLdap", "logMail", "logServer", "logScheduler", "logTransfer",
-		"logsKeepDays", "productionMode", "pwForceDigit", "pwForceLower",
-		"pwForceSpecial", "pwForceUpper", "pwLengthMin", "schemaTimestamp",
-		"repoChecked", "repoFeedback", "repoSkipVerify", "tokenExpiryHours"}
+		"imagerThumbWidth", "logApplication", "logBackup", "logCache",
+		"logCluster", "logCsv", "logImager", "logLdap", "logMail", "logServer",
+		"logScheduler", "logTransfer", "logsKeepDays", "productionMode",
+		"pwForceDigit", "pwForceLower", "pwForceSpecial", "pwForceUpper",
+		"pwLengthMin", "schemaTimestamp", "repoChecked", "repoFeedback",
+		"repoSkipVerify", "tokenExpiryHours"}
 )
 
 // store setters

data/data_file.go

@@ -14,6 +14,7 @@ import (
 	"r3/config"
 	"r3/db"
 	"r3/handler"
+	"r3/image"
 	"r3/schema"
 	"r3/tools"
 	"r3/types"
@@ -23,21 +24,32 @@ import (
 	"github.com/jackc/pgx/v4"
 )
 
-// returns path to downloadable file, if access is granted
-func GetFilePath(loginId int64, attributeId uuid.UUID, fileId uuid.UUID) (string, error) {
+func CanAccessFile(loginId int64, attributeId uuid.UUID) error {
 	cache.Schema_mx.RLock()
 	defer cache.Schema_mx.RUnlock()
 
 	attribute, exists := cache.AttributeIdMap[attributeId]
 	if !exists || !schema.IsContentFiles(attribute.Content) {
-		return "", errors.New("not a file attribute")
+		return errors.New("not a file attribute")
 	}
 
 	// check for authorized access, READ(1) for GET
 	if !authorizedAttribute(loginId, attributeId, 1) {
-		return "", errors.New(handler.ErrUnauthorized)
+		return errors.New(handler.ErrUnauthorized)
 	}
-	return filepath.Join(config.File.Paths.Files, attribute.Id.String(), fileId.String()), nil
+	return nil
+}
+
+// returns path to downloadable file
+func GetFilePath(attributeId uuid.UUID, fileId uuid.UUID) string {
+	return filepath.Join(config.File.Paths.Files, attributeId.String(),
+		fileId.String())
+}
+
+// returns path to thumbnail of downloadable file
+func GetFilePathThumb(attributeId uuid.UUID, fileId uuid.UUID) string {
+	return filepath.Join(config.File.Paths.Files, attributeId.String(),
+		fmt.Sprintf("%s.webp", fileId.String()))
 }
 
 // attempts to store file upload
@@ -134,6 +146,10 @@ func SetFile(loginId int64, attributeId uuid.UUID, fileId uuid.UUID,
 		return fileId, err
 	}
 
+	// create/update thumbnail - failure should not block progress
+	image.CreateThumbnail(fileId, filepath.Ext(part.FileName()), filePath,
+		GetFilePathThumb(attributeId, fileId), false)
+
 	// store file reference
 	if isNewFile {
 		if _, err := db.Pool.Exec(db.Ctx, fmt.Sprintf(`

db/embedded/embedded.go

@@ -97,7 +97,7 @@ func Backup(path string) error {
 		"-f", path,
 	}
 	cmd := exec.Command(filepath.Join(dbBin, "pg_dump"), args...)
-	addSysProgAttrs(cmd)
+	tools.CmdAddSysProgAttrs(cmd)
 	cmd.Env = append(cmd.Env, fmt.Sprintf("LC_MESSAGES=%s", locale))
 	cmd.Env = append(cmd.Env, fmt.Sprintf("PGPASSWORD=%s", config.File.Db.Pass))
 	return cmd.Run()
@@ -125,7 +125,7 @@ func execWaitFor(call string, args []string, waitFor []string, waitTime int) (st
 	ctx, _ := context.WithTimeout(context.Background(), time.Duration(waitTime)*time.Second)
 
 	cmd := exec.CommandContext(ctx, call, args...)
-	addSysProgAttrs(cmd)
+	tools.CmdAddSysProgAttrs(cmd)
 	cmd.Env = append(os.Environ(), fmt.Sprintf("LC_MESSAGES=%s", locale))
 
 	stdout, err := cmd.StdoutPipe()

db/upgrade/upgrade.go

@@ -110,9 +110,14 @@ var upgradeFunctions = map[string]func(tx pgx.Tx) (string, error){
 			ALTER TABLE instance.login_setting ALTER COLUMN pattern
 				TYPE instance.login_setting_pattern USING pattern::text::instance.login_setting_pattern;
 			
-			-- new config
+			-- new log context
+			ALTER TYPE instance.log_context ADD VALUE 'imager';
+			
+			-- new config options
 			INSERT INTO instance.config (name,value) VALUES ('filesKeepDaysDeleted','90');
 			INSERT INTO instance.config (name,value) VALUES ('filesKeepDaysUnassigned','90');
+			INSERT INTO instance.config (name,value) VALUES ('imagerThumbWidth','300');
+			INSERT INTO instance.config (name,value) VALUES ('logImager',2);
 
 			-- changes to fixed tokens
 			ALTER TYPE instance.token_fixed_context ADD VALUE 'client';

handler/data_download/data_download.go

@@ -34,7 +34,7 @@ func Handler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// parse getters
+	// parse other getters
 	attributeId, err := handler.ReadUuidGetterFromUrl(r, "attribute_id")
 	if err != nil {
 		handler.AbortRequest(w, context, err, handler.ErrGeneral)
@@ -46,11 +46,12 @@ func Handler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// get and serve file
-	filePath, err := data.GetFilePath(loginId, attributeId, fileId)
-	if err != nil {
-		handler.AbortRequest(w, context, err, handler.ErrGeneral)
+	// check file access
+	if err := data.CanAccessFile(loginId, attributeId); err != nil {
+		handler.AbortRequest(w, context, err, handler.ErrUnauthorized)
 		return
 	}
-	http.ServeFile(w, r, filePath)
+
+	// serve file
+	http.ServeFile(w, r, data.GetFilePath(attributeId, fileId))
 }

handler/data_download_thumb/data_download_thumb.go

@@ -0,0 +1,93 @@
+package data_download_thumb
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"r3/bruteforce"
+	"r3/config"
+	"r3/data"
+	"r3/handler"
+	"r3/image"
+	"r3/login/login_auth"
+	"strings"
+)
+
+var context = "data_download_thumb"
+
+func Handler(w http.ResponseWriter, r *http.Request) {
+
+	if blocked := bruteforce.Check(r); blocked {
+		handler.AbortRequestNoLog(w, handler.ErrBruteforceBlock)
+		return
+	}
+
+	// check if thumbnail processing is available
+	if !image.GetCanProcess() {
+		w.Write(handler.NoImage)
+		return
+	}
+
+	// get authentication token
+	token, err := handler.ReadGetterFromUrl(r, "token")
+	if err != nil {
+		handler.AbortRequest(w, context, err, handler.ErrGeneral)
+		return
+	}
+
+	// check token, any login is generally allowed to attempt a download
+	var loginId int64
+	var admin bool
+	var noAuth bool
+	if _, err := login_auth.Token(token, &loginId, &admin, &noAuth); err != nil {
+		handler.AbortRequest(w, context, err, handler.ErrAuthFailed)
+		bruteforce.BadAttempt(r)
+		return
+	}
+
+	// parse other getters
+	attributeId, err := handler.ReadUuidGetterFromUrl(r, "attribute_id")
+	if err != nil {
+		handler.AbortRequest(w, context, err, handler.ErrGeneral)
+		return
+	}
+	fileId, err := handler.ReadUuidGetterFromUrl(r, "file_id")
+	if err != nil {
+		handler.AbortRequest(w, context, err, handler.ErrGeneral)
+		return
+	}
+
+	// check file access
+	if err := data.CanAccessFile(loginId, attributeId); err != nil {
+		fmt.Println("bad access")
+		handler.AbortRequest(w, context, err, handler.ErrUnauthorized)
+		return
+	}
+
+	// check whether thumbnail file exists
+	filePath := data.GetFilePathThumb(attributeId, fileId)
+
+	_, err = os.Stat(filePath)
+	if err != nil && !os.IsNotExist(err) {
+		handler.AbortRequest(w, context, err, handler.ErrGeneral)
+		return
+	}
+
+	// thumbnail file does not exist, attempt to create it
+	if os.IsNotExist(err) {
+		fmt.Println("no existence, create")
+
+		urlElms := strings.Split(r.URL.Path, "/")
+		fileExt := filepath.Ext(urlElms[len(urlElms)-1])
+
+		filePathSrc := filepath.Join(config.File.Paths.Files,
+			attributeId.String(), fileId.String())
+
+		if err := image.CreateThumbnail(fileId, fileExt, filePathSrc, filePath, true); err != nil {
+			w.Write(handler.NoImage)
+			return
+		}
+	}
+	http.ServeFile(w, r, filePath)
+}

handler/handler.go

@@ -11,6 +11,10 @@ import (
 	"github.com/gofrs/uuid"
 )
 
+var (
+	NoImage []byte
+)
+
 func GetStringFromPart(part *multipart.Part) string {
 	buf := new(bytes.Buffer)
 	buf.ReadFrom(part)
@@ -44,6 +48,9 @@ func ReadGetterFromUrl(r *http.Request, name string) (string, error) {
 	}
 	return keys[0], nil
 }
+func SetNoImage(v []byte) {
+	NoImage = v
+}
 
 func AbortRequest(w http.ResponseWriter, context string, err error, errMessageUser string) {
 	log.Error("server", fmt.Sprintf("aborted %s request", context), err)

handler/handler_error.go

@@ -51,6 +51,7 @@ const (
 )
 
 var (
+	// errors
 	errContexts     = []string{"APP", "CSV", "DBS", "SEC"}
 	errCodeRx       = regexp.MustCompile(`^{ERR_([A-Z]{3})_(\d{3})}`)
 	errExpectedList = []errExpected{