From 324ffae2b0e663c1e62d84ecad5f56794f394558 Mon Sep 17 00:00:00 2001 From: Earle Nietzel Date: Wed, 14 Aug 2024 15:37:35 -0400 Subject: [PATCH] SAK-50375 Content calculate collection sizes by collection type (#12791) --- .../content/impl/BaseContentService.java | 43 +- .../content/impl/ContentServiceSql.java | 11 + .../impl/ContentServiceSqlDefault.java | 468 +++++++++--------- .../content/impl/DbContentService.java | 45 +- 4 files changed, 277 insertions(+), 290 deletions(-) diff --git a/kernel/kernel-impl/src/main/java/org/sakaiproject/content/impl/BaseContentService.java b/kernel/kernel-impl/src/main/java/org/sakaiproject/content/impl/BaseContentService.java index d7bec1fb6694..e4e0b465e8e1 100644 --- a/kernel/kernel-impl/src/main/java/org/sakaiproject/content/impl/BaseContentService.java +++ b/kernel/kernel-impl/src/main/java/org/sakaiproject/content/impl/BaseContentService.java @@ -10880,20 +10880,33 @@ public List getMembers() * * @return The size of all the resource body bytes within this collection in Kbytes. */ - public long getBodySizeK() - { - long size = 0; - - String context = getContext(); - if(context != null || m_id.startsWith(COLLECTION_DROPBOX)) - { - size = getSizeForContext(context!=null?context:m_id)/1000L; - } - - log.debug("getBodySizeK(): collection: {} size: {}",getId(),size); - return size; + public long getBodySizeK() { + long size = 0; + String context = getContext(); + if (context != null) { + Map sizes = getSizeForContext(context); + if (m_id.startsWith(COLLECTION_DROPBOX)) { + size = sizes.keySet().stream() + .filter(k -> k.startsWith(COLLECTION_DROPBOX)) + .mapToLong(k -> Long.valueOf(sizes.get(k))) + .sum(); + } else if (m_id.startsWith(COLLECTION_USER)) { + size = sizes.keySet().stream() + .filter(k -> k.startsWith(COLLECTION_USER)) + .mapToLong(k -> Long.valueOf(sizes.get(k))) + .sum(); + } else { + size = sizes.keySet().stream() + .filter(k -> k.startsWith(COLLECTION_SITE)) + .mapToLong(k -> Long.valueOf(sizes.get(k))) + .sum(); + } - } // getBodySizeK + if (size > 0) size /= 1024L; + } + log.debug("getBodySizeK(): collection: {} size: {}", getId(), size); + return size; + } /** * Access a List of the collections' internal members as full ContentResource or ContentCollection objects. @@ -12982,9 +12995,9 @@ public void registerSiteContentAdvisorProvidor(SiteContentAdvisorProvider adviso siteContentAdvisorsProviders.put(type, advisor); } - protected long getSizeForContext(String context) + public Map getSizeForContext(String context) { - return 0; + return Collections.emptyMap(); } public Map transferCopyEntities(String fromContext, String toContext, List ids, List options, boolean cleanup) { diff --git a/kernel/kernel-impl/src/main/java/org/sakaiproject/content/impl/ContentServiceSql.java b/kernel/kernel-impl/src/main/java/org/sakaiproject/content/impl/ContentServiceSql.java index ae94c494268c..c612aa3ed5c0 100644 --- a/kernel/kernel-impl/src/main/java/org/sakaiproject/content/impl/ContentServiceSql.java +++ b/kernel/kernel-impl/src/main/java/org/sakaiproject/content/impl/ContentServiceSql.java @@ -136,6 +136,17 @@ public interface ContentServiceSql * returns the sql statement which retrieves the total number of bytes within a site-level collection (context) in the CONTENT_RESOURCE table. */ String getQuotaQuerySql(); + + /** + * returns the sql statement which retrieves the total number of bytes for each collection within a site. + */ + String getContextSizesSql(); + + /** + * returns the sql statement which retrieves the total number of bytes for this collection + */ + String getCollectionSizeSql(); + String getDropBoxQuotaQuerySql(); /** * returns the sql statement which retrieves the total number of bytes within a site-level collection skiping user folders. diff --git a/kernel/kernel-impl/src/main/java/org/sakaiproject/content/impl/ContentServiceSqlDefault.java b/kernel/kernel-impl/src/main/java/org/sakaiproject/content/impl/ContentServiceSqlDefault.java index bf5509e9321d..11805686279c 100644 --- a/kernel/kernel-impl/src/main/java/org/sakaiproject/content/impl/ContentServiceSqlDefault.java +++ b/kernel/kernel-impl/src/main/java/org/sakaiproject/content/impl/ContentServiceSqlDefault.java @@ -1,8 +1,8 @@ -/********************************************************************************** - * $URL: https://source.sakaiproject.org/contrib/rsmart/dbrefactor/chat/chat-impl/impl/src/java/org/sakaiproject/chat/impl/ChatServiceSqlDefault.java $ - * $Id: ChatServiceSqlDefault.java 3560 2007-02-19 22:08:01Z jbush@rsmart.com $ - *********************************************************************************** - * +/********************************************************************************** + * $URL$ + * $Id$ + *********************************************************************************** + * * Copyright (c) 2007, 2008 Sakai Foundation * * Licensed under the Educational Community License, Version 2.0 (the "License"); @@ -16,242 +16,222 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * - **********************************************************************************/ - -package org.sakaiproject.content.impl; - - -/** - * methods for accessing content data in a database. - */ -public class ContentServiceSqlDefault implements ContentServiceSql -{ - /** - * returns the sql statement which retrieves the body from the specified table (content_resource_body_binary). - */ - public String getBodySql(String table) - { - return "select BODY from " + table + " where ( RESOURCE_ID = ? )"; - } - - /** - * returns the sql statement which retrieves the collection id from the specified table. - */ - public String getCollectionIdSql(String table) - { - return "select COLLECTION_ID from " + table + " where IN_COLLECTION = ?"; - } - - /** - * returns the sql statement which deletes content from the specified table (content_resource_body_binary). - */ - public String getDeleteContentSql(String table) - { - return "delete from " + table + " where resource_id = ? "; - } - - /** - * returns the sql statement which inserts content into the specified table (content_resource_body_binary). - */ - public String getInsertContentSql(String table) - { - return "insert into " + table + " (RESOURCE_ID, RESOURCE_SHA256, BODY)" + " values (? , ? , ? )"; - } - - /** - * returns the sql statement which retrieves the number of content resources from the content_resource table. - */ - public String getNumContentResources1Sql() - { - return "select count(IN_COLLECTION) from CONTENT_RESOURCE where IN_COLLECTION like ?"; - } - - /** - * returns the sql statement which retrieves the number of content resources from the content_collection table. - */ - public String getNumContentResources2Sql() - { - return "select count(IN_COLLECTION) from CONTENT_COLLECTION where IN_COLLECTION like ?"; - } - - /** - * returns the sql statement which retrieves the number of content resources from the content_resource table. - */ - public String getNumContentResources3Sql() - { - return "select count(IN_COLLECTION) from CONTENT_RESOURCE where IN_COLLECTION = ?"; - } - - /** - * returns the sql statement which retrieves the number of content resources from the content_collection table. - */ - public String getNumContentResources4Sql() - { - return "select count(IN_COLLECTION) from CONTENT_COLLECTION where IN_COLLECTION = ?"; - } - - /** - * returns the sql statement which retrieves the resource id from the content_resource table. - */ - public String getResourceId1Sql() - { - return "select RESOURCE_ID from CONTENT_RESOURCE where RESOURCE_UUID=?"; - } - - /** - * returns the sql statement which retrieves the resource id from the content_resource_body_binary table. - */ - public String getResourceId2Sql() - { - return "select RESOURCE_ID from CONTENT_RESOURCE_BODY_BINARY where (RESOURCE_ID = ?)"; - } - - /** - * returns the sql statement which retrieves the resource id from the specified table. - */ - public String getResourceId3Sql(String table) - { - return "select RESOURCE_ID from " + table + " where IN_COLLECTION = ?"; - } - - /** - * returns the sql statement which retrieves the resource id and xml fields from the content_resource table. - */ - public String getResourceIdXmlSql() - { - return "select RESOURCE_ID, XML, BINARY_ENTITY from CONTENT_RESOURCE where FILE_PATH IS NULL"; - } - - /** - * {@inheritDoc} - */ - public String getResourceIdAndFilePath() - { - return "select RESOURCE_ID, FILE_PATH from CONTENT_RESOURCE where FILE_PATH IS NOT NULL"; - } - - /** - * returns the sql statement which retrieves the resource uuid from the content_resource table. - */ - public String getResourceUuidSql() - { - return "select RESOURCE_UUID from CONTENT_RESOURCE where RESOURCE_ID=?"; - } - - /** - * returns the sql statement which updates the resource uuid in the content_resource table for a given resource uuid. - */ - public String getUpdateContentResource1Sql() - { - return "update CONTENT_RESOURCE set RESOURCE_UUID = ? where RESOURCE_UUID = ?"; - } - - /** - * returns the sql statement which updates the resource uuid in the content_resource table for a given resource id. - */ - public String getUpdateContentResource2Sql() - { - return "update CONTENT_RESOURCE set RESOURCE_UUID = ? where RESOURCE_ID = ?"; - } - - /** - * returns the sql statement which updates the file path and xml fields in the content_resource table for a given resource id. - */ - public String getUpdateContentResource3Sql() - { - return "update CONTENT_RESOURCE set FILE_PATH = ?, XML = NULL, BINARY_ENTITY = ?, CONTEXT = ?, FILE_SIZE = ?, RESOURCE_TYPE_ID = ? where RESOURCE_ID = ?"; - } - - /** - * returns the sql statement which retrieves pairs of individual-dropbox-id and last-update fields from the content_dropbox_changes table for a given site-level dropbox-id. - */ - public String getIndividualDropboxChangeSql() - { - return "select LAST_UPDATE from CONTENT_DROPBOX_CHANGES where (DROPBOX_ID = ?)"; - } - - /** - * returns the sql statement which retrieves the last-update field from the content_dropbox_changes table for a given individual-dropbox-id. - */ - public String getSiteDropboxChangeSql() - { - return "select DROPBOX_ID, LAST_UPDATE from CONTENT_DROPBOX_CHANGES where (IN_COLLECTION = ?)"; - } - - /** - * returns the sql statement which updates the last-update field in the content_dropbox_changes table for a given site-level dropbox-id and individual-dropbox-id. - */ - public String getUpdateIndividualDropboxChangeSql() - { - return "update CONTENT_DROPBOX_CHANGES set IN_COLLECTION = ?, LAST_UPDATE = ? where DROPBOX_ID = ?"; - } - - /** - * returns the sql statement which inserts the individual-dropbox-id, site-level dropbox-id and last-update fields into the content_dropbox_changes table. - */ - public String getInsertIndividualDropboxChangeSql() - { - return "insert into CONTENT_DROPBOX_CHANGES (DROPBOX_ID, IN_COLLECTION, LAST_UPDATE) values (? , ? , ?) on duplicate key update IN_COLLECTION = ?, LAST_UPDATE = ?"; - } - /** - * returns the sql statement which retrieves the total number of bytes within a site-level collection (context) in the CONTENT_RESOURCE table. - */ - public String getQuotaQuerySql() - { - return "select SUM(FILE_SIZE) from CONTENT_RESOURCE where CONTEXT = ?"; - } - /** - * returns the sql statement which retrieves the total number of bytes within a site-level collection (context) in the CONTENT_RESOURCE table. - */ - public String getDropBoxQuotaQuerySql() - { - return "select SUM(FILE_SIZE) from CONTENT_RESOURCE where IN_COLLECTION LIKE ?"; - } - - /** - * returns the sql statement which retrieves the RESOURCE_ID and XML values for all entries in the CONTENT_RESOURCE table where file-size is null. - */ - public String getAccessResourceIdAndXmlSql(String table) - { - return "select RESOURCE_ID, RESOURCE_UUID, XML from " + table + " where FILE_SIZE is NULL"; - } - - /* - * (non-Javadoc) - * - * @see org.sakaiproject.content.impl.ContentServiceSql#getCreateTemporaryUTF8TestTable(java.lang.String) - */ - public String getCreateTemporaryUTF8TestTable(String tempTableName) - { - return "create table " + tempTableName + " ( id int, bval varchar(2048) )"; - } - - /* - * (non-Javadoc) - * - * @see org.sakaiproject.content.impl.ContentServiceSql#getDropTemporaryUTF8TestTable(java.lang.String) - */ - public String getDropTemporaryUTF8TestTable(String tempTableName) - { - return "drop table " + tempTableName; - } - - /** - * returns the sql statement which retrieves the BINARY_ENTITY and XML values for all entries in the CONTENT_RESOURCE table, - * selecting by the RESOURCE_TYPE_ID with first and last record indexes, and returned in ascending order by RESOURCE_ID. - */ - public String getSelectByResourceTypeQuerySql() - { - return "select BINARY_ENTITY, XML from CONTENT_RESOURCE where RESOURCE_TYPE_ID = ? ORDER BY RESOURCE_ID LIMIT ?, ? "; - } - - /** - * returns the sql statement which retrieves the total number of bytes within a site-level collection skiping user folders. - * KNL-1084, SAK-22169 - */ - public String getDropBoxRootQuotaQuerySql() { - return "select SUM(FILE_SIZE) from CONTENT_RESOURCE where IN_COLLECTION LIKE ? and not exists (select 1 from SAKAI_USER_ID_MAP where USER_ID = substr(in_collection,length(?)+1,instr(substr(in_collection,length(?)+1),'/')-1))"; - } - -} + * + **********************************************************************************/ + +package org.sakaiproject.content.impl; + + +/** + * methods for accessing content data in a database. + */ +public class ContentServiceSqlDefault implements ContentServiceSql { + /** + * returns the sql statement which retrieves the body from the specified table (content_resource_body_binary). + */ + public String getBodySql(String table) { + return "select BODY from " + table + " where ( RESOURCE_ID = ? )"; + } + + /** + * returns the sql statement which retrieves the collection id from the specified table. + */ + public String getCollectionIdSql(String table) { + return "select COLLECTION_ID from " + table + " where IN_COLLECTION = ?"; + } + + /** + * returns the sql statement which deletes content from the specified table (content_resource_body_binary). + */ + public String getDeleteContentSql(String table) { + return "delete from " + table + " where resource_id = ? "; + } + + /** + * returns the sql statement which inserts content into the specified table (content_resource_body_binary). + */ + public String getInsertContentSql(String table) { + return "insert into " + table + " (RESOURCE_ID, RESOURCE_SHA256, BODY)" + " values (? , ? , ? )"; + } + + /** + * returns the sql statement which retrieves the number of content resources from the content_resource table. + */ + public String getNumContentResources1Sql() { + return "select count(IN_COLLECTION) from CONTENT_RESOURCE where IN_COLLECTION like ?"; + } + + /** + * returns the sql statement which retrieves the number of content resources from the content_collection table. + */ + public String getNumContentResources2Sql() { + return "select count(IN_COLLECTION) from CONTENT_COLLECTION where IN_COLLECTION like ?"; + } + + /** + * returns the sql statement which retrieves the number of content resources from the content_resource table. + */ + public String getNumContentResources3Sql() { + return "select count(IN_COLLECTION) from CONTENT_RESOURCE where IN_COLLECTION = ?"; + } + + /** + * returns the sql statement which retrieves the number of content resources from the content_collection table. + */ + public String getNumContentResources4Sql() { + return "select count(IN_COLLECTION) from CONTENT_COLLECTION where IN_COLLECTION = ?"; + } + + /** + * returns the sql statement which retrieves the resource id from the content_resource table. + */ + public String getResourceId1Sql() { + return "select RESOURCE_ID from CONTENT_RESOURCE where RESOURCE_UUID=?"; + } + + /** + * returns the sql statement which retrieves the resource id from the content_resource_body_binary table. + */ + public String getResourceId2Sql() { + return "select RESOURCE_ID from CONTENT_RESOURCE_BODY_BINARY where (RESOURCE_ID = ?)"; + } + + /** + * returns the sql statement which retrieves the resource id from the specified table. + */ + public String getResourceId3Sql(String table) { + return "select RESOURCE_ID from " + table + " where IN_COLLECTION = ?"; + } + + /** + * returns the sql statement which retrieves the resource id and xml fields from the content_resource table. + */ + public String getResourceIdXmlSql() { + return "select RESOURCE_ID, XML, BINARY_ENTITY from CONTENT_RESOURCE where FILE_PATH IS NULL"; + } + + public String getResourceIdAndFilePath() { + return "select RESOURCE_ID, FILE_PATH from CONTENT_RESOURCE where FILE_PATH IS NOT NULL"; + } + + /** + * returns the sql statement which retrieves the resource uuid from the content_resource table. + */ + public String getResourceUuidSql() { + return "select RESOURCE_UUID from CONTENT_RESOURCE where RESOURCE_ID=?"; + } + + /** + * returns the sql statement which updates the resource uuid in the content_resource table for a given resource uuid. + */ + public String getUpdateContentResource1Sql() { + return "update CONTENT_RESOURCE set RESOURCE_UUID = ? where RESOURCE_UUID = ?"; + } + + /** + * returns the sql statement which updates the resource uuid in the content_resource table for a given resource id. + */ + public String getUpdateContentResource2Sql() { + return "update CONTENT_RESOURCE set RESOURCE_UUID = ? where RESOURCE_ID = ?"; + } + + /** + * returns the sql statement which updates the file path and xml fields in the content_resource table for a given resource id. + */ + public String getUpdateContentResource3Sql() { + return "update CONTENT_RESOURCE set FILE_PATH = ?, XML = NULL, BINARY_ENTITY = ?, CONTEXT = ?, FILE_SIZE = ?, RESOURCE_TYPE_ID = ? where RESOURCE_ID = ?"; + } + + /** + * returns the sql statement which retrieves pairs of individual-dropbox-id and last-update fields from the content_dropbox_changes table for a given site-level dropbox-id. + */ + public String getIndividualDropboxChangeSql() { + return "select LAST_UPDATE from CONTENT_DROPBOX_CHANGES where (DROPBOX_ID = ?)"; + } + + /** + * returns the sql statement which retrieves the last-update field from the content_dropbox_changes table for a given individual-dropbox-id. + */ + public String getSiteDropboxChangeSql() { + return "select DROPBOX_ID, LAST_UPDATE from CONTENT_DROPBOX_CHANGES where (IN_COLLECTION = ?)"; + } + + /** + * returns the sql statement which updates the last-update field in the content_dropbox_changes table for a given site-level dropbox-id and individual-dropbox-id. + */ + public String getUpdateIndividualDropboxChangeSql() { + return "update CONTENT_DROPBOX_CHANGES set IN_COLLECTION = ?, LAST_UPDATE = ? where DROPBOX_ID = ?"; + } + + /** + * returns the sql statement which inserts the individual-dropbox-id, site-level dropbox-id and last-update fields into the content_dropbox_changes table. + */ + public String getInsertIndividualDropboxChangeSql() { + return "insert into CONTENT_DROPBOX_CHANGES (DROPBOX_ID, IN_COLLECTION, LAST_UPDATE) values (? , ? , ?) on duplicate key update IN_COLLECTION = ?, LAST_UPDATE = ?"; + } + + /** + * returns the sql statement which retrieves the total number of bytes within a site-level collection (context) in the CONTENT_RESOURCE table. + */ + public String getQuotaQuerySql() { + return "select SUM(FILE_SIZE) from CONTENT_RESOURCE where CONTEXT = ?"; + } + + @Override + public String getContextSizesSql() { + return "select IN_COLLECTION, SUM(FILE_SIZE) from CONTENT_RESOURCE where CONTEXT = ? GROUP BY IN_COLLECTION"; + } + + @Override + public String getCollectionSizeSql() { + return "select SUM(FILE_SIZE) from CONTENT_RESOURCE where IN_COLLECTION LIKE = ?"; + } + + /** + * returns the sql statement which retrieves the total number of bytes within a site-level collection (context) in the CONTENT_RESOURCE table. + */ + public String getDropBoxQuotaQuerySql() { + return "select SUM(FILE_SIZE) from CONTENT_RESOURCE where IN_COLLECTION LIKE ?"; + } + + /** + * returns the sql statement which retrieves the RESOURCE_ID and XML values for all entries in the CONTENT_RESOURCE table where file-size is null. + */ + public String getAccessResourceIdAndXmlSql(String table) { + return "select RESOURCE_ID, RESOURCE_UUID, XML from " + table + " where FILE_SIZE is NULL"; + } + + /* + * (non-Javadoc) + * + * @see org.sakaiproject.content.impl.ContentServiceSql#getCreateTemporaryUTF8TestTable(java.lang.String) + */ + public String getCreateTemporaryUTF8TestTable(String tempTableName) { + return "create table " + tempTableName + " ( id int, bval varchar(2048) )"; + } + + /* + * (non-Javadoc) + * + * @see org.sakaiproject.content.impl.ContentServiceSql#getDropTemporaryUTF8TestTable(java.lang.String) + */ + public String getDropTemporaryUTF8TestTable(String tempTableName) { + return "drop table " + tempTableName; + } + + /** + * returns the sql statement which retrieves the BINARY_ENTITY and XML values for all entries in the CONTENT_RESOURCE table, + * selecting by the RESOURCE_TYPE_ID with first and last record indexes, and returned in ascending order by RESOURCE_ID. + */ + public String getSelectByResourceTypeQuerySql() { + return "select BINARY_ENTITY, XML from CONTENT_RESOURCE where RESOURCE_TYPE_ID = ? ORDER BY RESOURCE_ID LIMIT ?, ? "; + } + + /** + * returns the sql statement which retrieves the total number of bytes within a site-level collection skiping user folders. + * KNL-1084, SAK-22169 + */ + public String getDropBoxRootQuotaQuerySql() { + return "select SUM(FILE_SIZE) from CONTENT_RESOURCE where IN_COLLECTION LIKE ? and not exists (select 1 from SAKAI_USER_ID_MAP where USER_ID = substr(in_collection,length(?)+1,instr(substr(in_collection,length(?)+1),'/')-1))"; + } +} diff --git a/kernel/kernel-impl/src/main/java/org/sakaiproject/content/impl/DbContentService.java b/kernel/kernel-impl/src/main/java/org/sakaiproject/content/impl/DbContentService.java index a9274cf36c1d..3d732b2f88cc 100644 --- a/kernel/kernel-impl/src/main/java/org/sakaiproject/content/impl/DbContentService.java +++ b/kernel/kernel-impl/src/main/java/org/sakaiproject/content/impl/DbContentService.java @@ -66,7 +66,6 @@ import org.sakaiproject.content.impl.serialize.impl.conversion.Type1BlobCollectionConversionHandler; import org.sakaiproject.db.api.SqlReader; import org.sakaiproject.db.api.SqlService; -import org.sakaiproject.entity.api.Entity; import org.sakaiproject.entity.api.ResourceProperties; import org.sakaiproject.entity.api.ResourcePropertiesEdit; import org.sakaiproject.entity.api.serialize.EntityParseException; @@ -3074,38 +3073,22 @@ public Object readSqlResultRecord(ResultSet result) } - protected long getSizeForContext(String context) - { - long size = 0L; - - String sql = contentServiceSql.getQuotaQuerySql(); - Object[] fields = new Object[] { context.startsWith(COLLECTION_DROPBOX) ? StorageUtils.escapeSqlLike(context) + "%" : context }; - if (context.startsWith(COLLECTION_DROPBOX)) { - if (context.split(Entity.SEPARATOR).length == 4) { - // User Folder - sql = contentServiceSql.getDropBoxQuotaQuerySql(); - } else { - // Root Folder - KNL-1084, SAK-22169 - fields = new Object[] { StorageUtils.escapeSqlLike(context) +"%", context, context }; - sql = contentServiceSql.getDropBoxRootQuotaQuerySql(); - } - } - - List list = sqlService.dbRead(sql, fields, null); - if(list != null && ! list.isEmpty()) - { - String result = (String) list.get(0); - try - { - size = Float.valueOf(result).longValue(); - } - catch(Exception e) - { - log.warn("getSizeForContext() unable to parse long from \"" + result + "\" for context \"" + context + "\""); + public Map getSizeForContext(String context) { + Map sizes = new HashMap<>(); + Object[] fields = new Object[] {context}; + String sql = contentServiceSql.getContextSizesSql(); + SqlReader sqlReader = result -> { + // no return value is needed as the result is added to sizes (closure) + try { + sizes.put(result.getString(1), result.getLong(2)); + } catch (SQLException e) { + log.warn("calculating collection sizes, {}", e.toString()); } - } + return null; + }; - return size; + sqlService.dbRead(sql, fields, sqlReader); + return sizes; } /**