Customizable File Templates for Sources, Sample Types & Assay Designs

api/src/org/labkey/api/data/AbstractTableInfo.java

@@ -61,6 +61,7 @@
 import org.labkey.api.security.permissions.Permission;
 import org.labkey.api.security.permissions.ReadPermission;
 import org.labkey.api.sql.LabKeySql;
+import org.labkey.api.study.assay.FileLinkDisplayColumn;
 import org.labkey.api.util.ConfigurationException;
 import org.labkey.api.util.ContainerContext;
 import org.labkey.api.util.MemTrackable;
@@ -2054,6 +2055,24 @@ else if (pair.second instanceof StringExpressionFactory.URLStringExpression expr
         return templates;
     }
 
+    @Override
+    public List<Pair<String, String>> getValidatedImportTemplates(ViewContext ctx)
+    {
+        List<Pair<String, String>> templates = new ArrayList<>();
+        List<Pair<String, String>> allTemplates = getImportTemplates(ctx);
+        for (Pair<String, String> template : allTemplates)
+        {
+            if (template.second.toLowerCase().contains("exportexceltemplate"))
+                templates.add(template);
+            else
+            {
+                boolean fileExist = FileLinkDisplayColumn.filePathExist(template.second, ctx.getContainer(), ctx.getUser());
+                templates.add(new Pair<>(template.first, template.second + (fileExist ? "" : " (unavailable)")));
+            }
+        }
+        return templates;
+    }
+
     @Override
     public List<Pair<String, StringExpression>> getRawImportTemplates()
     {

api/src/org/labkey/api/data/TableInfo.java

@@ -401,6 +401,11 @@ default Set<FieldKey> getMethodRequiredFieldKeys()
      */
     List<Pair<String, String>> getImportTemplates(ViewContext ctx);
 
+    default List<Pair<String, String>> getValidatedImportTemplates(ViewContext ctx)
+    {
+        return getImportTemplates(ctx);
+    }
+
     /**
      * Returns a list of the raw import templates (without substituting the container).  This is intended to be
      * used by FilteredTable or other instances that need to copy the raw values from a parent table.  In general,

api/src/org/labkey/api/exp/api/SampleTypeDomainKind.java

@@ -84,6 +84,7 @@ public class SampleTypeDomainKind extends AbstractDomainKind<SampleTypeDomainKin
     private static final Logger logger;
     public static final String NAME = "SampleSet";
     public static final String PROVISIONED_SCHEMA_NAME = "expsampleset";
+    public static final String SAMPLETYPE_FILE_DIRECTORY = "sampletype";
 
     private static final Set<PropertyStorageSpec> BASE_PROPERTIES;
     private static final Set<PropertyStorageSpec.Index> INDEXES;
@@ -677,4 +678,10 @@ public boolean supportsNamingPattern()
     {
         return true;
     }
+
+    @Override
+    public String getDomainFileDirectory()
+    {
+        return SAMPLETYPE_FILE_DIRECTORY;
+    }
 }

api/src/org/labkey/api/exp/property/DomainKind.java

@@ -415,4 +415,9 @@ public boolean supportsNamingPattern()
     {
         return false;
     }
+
+    public String getDomainFileDirectory()
+    {
+        return getKindName();
+    }
 }

api/src/org/labkey/api/query/QueryDefinition.java

@@ -39,6 +39,14 @@
 
 public interface QueryDefinition
 {
+    String DEFAULT_METADATA_TEXT =
+            "<tables xmlns=\"http://labkey.org/data/xml\">\n" +
+                    "  <table tableName=\"%s\" tableDbType=\"NOT_IN_DB\">\n" +
+                    "    <columns>\n" +
+                    "    </columns>\n" +
+                    "  </table>\n" +
+                    "</tables>\n";
+
     String getName();
     void setName(String name);
     String getTitle();

api/src/org/labkey/api/settings/ProductFeature.java

@@ -11,6 +11,7 @@ public enum ProductFeature
     BiologicsRegistry("biologics"),
     CalculatedFields("core"),
     ChartBuilding("core"),
+    CustomImportTemplates("core"),
     DataChangeCommentRequirement("core"),
     ELN("labbook"),
     FreezerManagement("inventory"),

api/src/org/labkey/api/study/assay/FileLinkDisplayColumn.java

@@ -31,16 +31,22 @@
 import org.labkey.api.data.TableViewForm;
 import org.labkey.api.exp.PropertyDescriptor;
 import org.labkey.api.files.FileContentService;
+import org.labkey.api.pipeline.PipeRoot;
 import org.labkey.api.pipeline.PipelineService;
 import org.labkey.api.query.DetailsURL;
 import org.labkey.api.query.FieldKey;
 import org.labkey.api.query.SchemaKey;
+import org.labkey.api.security.User;
+import org.labkey.api.security.permissions.ReadPermission;
 import org.labkey.api.util.ContainerContext;
 import org.labkey.api.util.FileUtil;
 import org.labkey.api.util.NetworkDrive;
 import org.labkey.api.util.PageFlowUtil;
+import org.labkey.api.util.Path;
 import org.labkey.api.util.URIUtil;
 import org.labkey.api.view.ActionURL;
+import org.labkey.api.webdav.WebdavResource;
+import org.labkey.api.webdav.WebdavService;
 
 import java.io.File;
 import java.io.FileInputStream;
@@ -248,6 +254,39 @@ protected String getFileName(RenderContext ctx, Object value)
         return getFileName(ctx, value, false);
     }
 
+    public static boolean filePathExist(String path, Container container, User user)
+    {
+        String davPath = path;
+        if (FileUtil.isUrlEncoded(davPath))
+            davPath = FileUtil.decodeURL(davPath);
+        var resolver = WebdavService.get().getResolver();
+        // Resolve path under webdav root
+        Path parsed = Path.parse(StringUtils.trim(davPath));
+        WebdavResource resource = resolver.lookup(parsed);
+        if ((null == resource || !resource.exists()) && !parsed.startsWith(new Path("_webdav")))
+            resource = resolver.lookup(new Path("_webdav").append(parsed));
+        if (resource != null && resource.isFile() && resource.canRead(user, true))
+        {
+            return true;
+        }
+        else
+        {
+            // Resolve file under pipeline root
+            PipeRoot root = PipelineService.get().findPipelineRoot(container);
+            if (root != null)
+            {
+                // Attempt absolute path first, then relative path from pipeline root
+                File f = new File(path);
+                if (!root.isUnderRoot(f))
+                    f = root.resolvePath(path);
+
+                return (NetworkDrive.exists(f) && root.isUnderRoot(f) && root.hasPermission(container, user, ReadPermission.class));
+            }
+        }
+
+        return false;
+    }
+
     @Override
     protected String getFileName(RenderContext ctx, Object value, boolean isDisplay)
     {

api/src/org/labkey/api/util/FileUtil.java

@@ -332,7 +332,7 @@ static String isAllowedFileName(String s, boolean checkFileExtension, AppProps a
         return null;
     }
 
-    private static @Nullable String validateFileName(String s)
+    public static @Nullable String validateFileName(String s)
     {
         return StringUtilsLabKey.validateLegalNames(s, restrictedPrintable, "Filename");
     }
@@ -1264,27 +1264,20 @@ public static String encodeForURL(String str, boolean checkEncoded)
 
         // str is unencoded; we need certain special chars encoded for it to become a URL
         // % & # @ ~ {} []
-        StringBuilder res = new StringBuilder();
-        for (int i = 0; i < str.length(); i++) {
-            if ('%' == str.charAt(i)) res.append("%25");
-            else if ('#' == str.charAt(i)) res.append("%23");
-            else if ('&' == str.charAt(i)) res.append("%26");
-            else if ('@' == str.charAt(i)) res.append("%40");
-            else if ('~' == str.charAt(i)) res.append("%7E");
-            else if ('{' == str.charAt(i)) res.append("%7B");
-            else if ('}' == str.charAt(i)) res.append("%7D");
-            else if ('[' == str.charAt(i)) res.append("%5B");
-            else if (']' == str.charAt(i)) res.append("%5D");
-            else if ('+' == str.charAt(i)) res.append("%2B");
-            else if (' ' == str.charAt(i)) res.append("%20");   // space also
-            else res.append(str.charAt(i));
-        }
-        return res.toString();
+        return StringUtils.replaceEach(str, DECODED, ENCODED);
+    }
+
+    private static final String[] ENCODED = {"%25", "%23", "%26", "%40", "%7E", "%7B", "%7D", "%5B", "%5D", "%2B", "%20"};
+    private static final String[] DECODED = {"%", "#", "&", "@", "~", "{", "}", "[", "]", "+", " "};
+
+    static public String decodeURL(String str)
+    {
+        return StringUtils.replaceEach(str, ENCODED, DECODED);
     }
 
     public static boolean isUrlEncoded(String str)
     {
-        return StringUtils.indexOfAny(str, new String[]{"%25", "%23", "%26", "%40", "%7E", "%7B", "%7D", "%5B", "%5D", "%2B", "%20"}) > -1;
+        return StringUtils.indexOfAny(str, ENCODED) > -1;
     }
 
     static boolean startsWith(String s, char ch)

api/src/org/labkey/api/util/JavaScriptFragment.java

@@ -4,6 +4,16 @@
 import org.apache.commons.lang3.StringUtils;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+import org.labkey.api.collections.CaseInsensitiveHashSet;
+import org.labkey.api.data.JavaScriptDisplayColumn;
+import org.labkey.api.view.UnauthorizedException;
+
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+import java.io.StringReader;
+import java.util.Collections;
+import java.util.Set;
 
 /**
  * Used to assert that a character sequence is valid, properly encoded JavaScript. Similar to HtmlString, though this class
@@ -64,6 +74,62 @@ public class JavaScriptFragment implements SafeToRender
         }
     }
 
+    private static final Set<String> DISALLOWED_SCRIPT_ELEMENTS = Collections.unmodifiableSet(new CaseInsensitiveHashSet("onClick", "onRender", "includeScript"));
+    private static final String CLASS_NAME_ELEMENT = "className";
+    public static void ensureXMLMetadataNoJavaScript(String metadataText)
+    {
+        try
+        {
+            XMLInputFactory inputFactory = XMLInputFactory.newInstance();
+            XMLStreamReader reader = inputFactory.createXMLStreamReader(new StringReader(metadataText));
+
+            // Issue 48660 - disallow JavaScriptDisplayColumn for non-developers
+            // When we're inside a <className> element, accumulate the contents to check when we hit the closing tag
+            StringBuilder className = null;
+
+            while (reader.hasNext())
+            {
+                reader.next();
+                if (reader.isStartElement())
+                {
+                    String localPath = reader.getName().getLocalPart();
+                    // These three elements directly include JavaScript or pointers to script files
+                    if (DISALLOWED_SCRIPT_ELEMENTS.contains(localPath))
+                    {
+                        throw new UnauthorizedException("Illegal element <" + localPath + ">. For permissions to use this element, contact your system administrator");
+                    }
+                    if (CLASS_NAME_ELEMENT.equalsIgnoreCase(localPath))
+                    {
+                        className = new StringBuilder();
+                    }
+                }
+
+                if (reader.isCharacters() && className != null)
+                {
+                    // Accumulate the content of the <className>
+                    className.append(reader.getText());
+                }
+
+                if (reader.isEndElement())
+                {
+                    String localPath = reader.getName().getLocalPart();
+                    if (CLASS_NAME_ELEMENT.equalsIgnoreCase(localPath) && className != null)
+                    {
+                        if (className.toString().contains(JavaScriptDisplayColumn.class.getName()))
+                        {
+                            throw new UnauthorizedException("For permissions to use JavaScriptDisplayColumn, contact your system administrator");
+                        }
+                        className = null;
+                    }
+                }
+            }
+        }
+        catch (XMLStreamException ignored)
+        {
+            // Let other XML validation and error feedback handle malformed XML
+        }
+    }
+
     // Callers use factory method unsafe() instead
     private JavaScriptFragment(String s)
     {

assay/api-src/org/labkey/api/assay/AssayResultDomainKind.java

@@ -34,6 +34,7 @@
 import java.util.HashSet;
 import java.util.Set;
 
+import static org.labkey.api.assay.AssayFileWriter.DIR_NAME;
 import static org.labkey.api.data.Table.CREATED_BY_COLUMN_NAME;
 import static org.labkey.api.data.Table.CREATED_COLUMN_NAME;
 import static org.labkey.api.data.Table.MODIFIED_BY_COLUMN_NAME;
@@ -176,4 +177,10 @@ public void deletePropertyDescriptor(Domain domain, User user, PropertyDescripto
 
         pair.first.removeFilterCriteriaForProperty(pd);
     }
+
+    @Override
+    public String getDomainFileDirectory()
+    {
+        return DIR_NAME;
+    }
 }

experiment/src/org/labkey/experiment/api/DataClassDomainKind.java

@@ -82,6 +82,7 @@ public class DataClassDomainKind extends AbstractDomainKind<DataClassDomainKindP
     private static final Logger LOG = LogHelper.getLogger(DataClassDomainKind.class, "Data class domain kind changes");
     public static final String NAME = "DataClass";
     public static final String PROVISIONED_SCHEMA_NAME = "expdataclass";
+    public static final String DATACLASS_FILE_DIRECTORY = "dataclass";
 
     private static final Set<PropertyStorageSpec> BASE_PROPERTIES;
     private static final Set<PropertyStorageSpec.Index> INDEXES;
@@ -491,4 +492,10 @@ public boolean supportsNamingPattern()
     {
         return true;
     }
+
+    @Override
+    public String getDomainFileDirectory()
+    {
+        return DATACLASS_FILE_DIRECTORY;
+    }
 }

experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java

@@ -110,6 +110,7 @@
 import java.util.stream.Collectors;
 
 import static java.util.Objects.requireNonNull;
+import static org.labkey.api.exp.api.SampleTypeDomainKind.SAMPLETYPE_FILE_DIRECTORY;
 import static org.labkey.api.util.StringExpressionFactory.AbstractStringExpression.NullValueBehavior.NullResult;
 import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.*;
 
@@ -1576,7 +1577,7 @@ public DataIteratorBuilder persistRows(DataIteratorBuilder data, DataIteratorCon
         try
         {
             var persist = new ExpDataIterators.PersistDataIteratorBuilder(data, this, propertiesTable, _ss, getUserSchema().getContainer(), getUserSchema().getUser(), _ss.getImportAliases(), sampleTypeObjectId)
-                    .setFileLinkDirectory("sampletype");
+                    .setFileLinkDirectory(SAMPLETYPE_FILE_DIRECTORY);
             SearchService searchService = SearchService.get();
             ExperimentServiceImpl experimentServiceImpl = ExperimentServiceImpl.get();
 

experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java

@@ -9045,9 +9045,37 @@ public Map<String, Map<String, Object>> getDomainMetrics()
         Map<String, Map<String, Object>> metrics = new HashMap<>();
         metrics.put("nameexpression", getNameExpressionMetrics());
         metrics.put("parentalias", getParentAliasMetrics());
+        metrics.put("importTemplates", getImportTemplatesMetrics());
         return metrics;
     }
 
+    private Map<String, Object> getImportTemplatesMetrics()
+    {
+        DbSchema dbSchema = CoreSchema.getInstance().getSchema();
+        SQLFragment sql = new SQLFragment("SELECT schema, metadata FROM query.querydef WHERE metadata LIKE '%<template%'");
+        Map<String, Object>[] results = new SqlSelector(dbSchema, sql).getMapArray();
+        Map<String, Long> counts = new HashMap<>();
+        final String sectionStart = "<importtemplates>";
+        for (Map<String, Object> result : results)
+        {
+            String schema = (String) result.get("schema");
+            String metadata = (String) result.get("metadata");
+            metadata = metadata.toLowerCase();
+            if (schema.toLowerCase().startsWith("assay.general."))
+                schema = "assay";
+            if (schema.equals("assay") || schema.equals("exp.data") || schema.equals("samples"))
+            {
+                if (!metadata.contains(sectionStart))
+                    continue;
+                long count = counts.get(schema) == null ? 0L : counts.get(schema);
+                counts.put(schema, ++count);
+            }
+        }
+        return Map.of("SampleType", counts.getOrDefault("samples", 0L),
+                "DataClass", counts.getOrDefault("exp.data", 0L),
+                "AssayDesign", counts.getOrDefault("assay", 0L));
+    }
+
     private Pair<Long, Long> getParentAliasMetrics(TableInfo tableInfo, String aliasField)
     {
         SQLFragment sql = new SQLFragment("SELECT ")

query/src/org/labkey/query/MetadataTableJSON.java

@@ -596,7 +596,7 @@ private static void addImportAliases(ColumnType xmlColumn, String importAliases)
     }
 
     @Nullable
-    private static TableType getTableType(String name, TablesDocument doc)
+    public static TableType getTableType(String name, TablesDocument doc)
     {
         if (doc != null && doc.getTables() != null)
         {
@@ -949,7 +949,7 @@ private static Pair<Lookup, Boolean> createLookup(ForeignKey fk, Container curre
     }
 
     @Nullable
-    private static TablesDocument parseDocument(String xml) throws XmlException
+    public static TablesDocument parseDocument(String xml) throws XmlException
     {
         if (xml == null)
         {