Add TableSize collection

Add table size collection & associated GUI changes.
Add pickers to custom reports

DBADash/CollectionCommandTimeout.cs

@@ -10,7 +10,7 @@ internal static class CollectionCommandTimeout
     {
         private static CommandTimeoutSettings settings = new();
 
-        public static int GetDefaultCommandTimeout()=>settings.DefaultCommandTimeout;
+        public static int GetDefaultCommandTimeout() => settings.DefaultCommandTimeout;
 
         static CollectionCommandTimeout()
         {
@@ -46,7 +46,7 @@ private static void LoadCommandTimeouts()
                     }
                 };
                 settings = JsonConvert.DeserializeObject<CommandTimeoutSettings>(json, serializationOpts);
-                
+
                 Log.Information($"Custom command timeouts loaded from {filePath}");
             }
             catch (JsonException ex)
@@ -68,6 +68,7 @@ private class CommandTimeoutSettings
             // Defaults supplied in this class will be used unless user supplies value in commandTimeouts.json
             public Dictionary<CollectionType, int> CollectionCommandTimeouts { get; } = new()
             {
+                { CollectionType.TableSize, 900 },
                 { CollectionType.DatabasePermissions, 900 },
                 { CollectionType.DatabasePrincipals, 900 },
                 { CollectionType.DatabaseRoleMembers, 900 },
@@ -108,10 +109,8 @@ private class CommandTimeoutSettings
                 { CollectionType.AvailabilityReplicas, 120 },
                 { CollectionType.MemoryUsage, 120 }
             };
-            public int DefaultCommandTimeout { get; } = 60;
 
+            public int DefaultCommandTimeout { get; } = 60;
         }
     }
-
-
 }

DBADash/CollectionSchedule.cs

@@ -10,6 +10,7 @@ public class CollectionSchedules : Dictionary<CollectionType, CollectionSchedule
         private const string hourly = "0 0 * ? * *";
         private const string midnight = "0 0 0 1/1 * ? *";
         private const string elevenPm = "0 0 23 1/1 * ? *";
+        private const string disabled = "";
 
         private static readonly CollectionSchedules collectionSchedules = new() {
                             {CollectionType.ServerProperties, new CollectionSchedule(){ Schedule = hourly } },
@@ -61,7 +62,9 @@ public class CollectionSchedules : Dictionary<CollectionType, CollectionSchedule
                             {CollectionType.ResourceGovernorConfiguration, new CollectionSchedule(){ Schedule = midnight } },
                             {CollectionType.DatabaseQueryStoreOptions, new CollectionSchedule(){ Schedule = midnight } },
                             {CollectionType.IdentityColumns, new CollectionSchedule(){ Schedule = midnight} },
-                            {CollectionType.SchemaSnapshot, new CollectionSchedule(){Schedule=elevenPm} }
+                            {CollectionType.SchemaSnapshot, new CollectionSchedule(){Schedule=elevenPm} },
+
+                            { CollectionType.TableSize, new CollectionSchedule() {Schedule = disabled, RunOnServiceStart = false} }
                   };
 
         public static readonly CollectionSchedules DefaultSchedules

DBADash/DBADash.csproj

@@ -23,6 +23,7 @@
     <None Remove="SQL\SQLJobs.sql" />
     <None Remove="SQL\SQLJobSteps.sql" />
     <None Remove="SQL\SQLRunningJobs.sql" />
+    <None Remove="SQL\SQLTableSize.sql" />
   </ItemGroup>
   <ItemGroup>
     <Compile Include="..\GlobalAssemblyInfo.cs" Link="Properties\GlobalAssemblyInfo.cs" />
@@ -82,6 +83,7 @@
 	<EmbeddedResource Include="SQL\SQLStopEventSessions.sql" />
 	<EmbeddedResource Include="SQL\SQLStopEventSessionsAzure.sql" />
 	<EmbeddedResource Include="SQL\SQLSysConfig.sql" />
+	<EmbeddedResource Include="SQL\SQLTableSize.sql" />
 	<EmbeddedResource Include="SQL\SQLTraceFlags.sql" />
 	<EmbeddedResource Include="SQL\SQLVLF.sql" />
 	<EmbeddedResource Include="SQL\SQLWaits.sql" />

DBADash/DBADashSource.cs

@@ -184,9 +184,18 @@ public bool PersistXESessions
 
         public bool CollectSessionWaits
         {
-            get => SourceConnection is { Type: ConnectionType.SQL } && _collectSessionWaits; set => _collectSessionWaits = value;
+            get => SourceConnection is { Type: ConnectionType.SQL } && _collectSessionWaits;
+            set => _collectSessionWaits = value;
         }
 
+        public int? TableSizeCollectionThresholdMB { get; set; }
+
+        public int? TableSizeMaxDatabaseThreshold { get; set; }
+
+        public string TableSizeDatabases { get; set; }
+
+        public int? TableSizeMaxTableThreshold { get; set; }
+
         public DBADashSource(string source)
         {
             this.ConnectionString = source;

DBADash/DBCollector.cs

@@ -13,6 +13,7 @@
 using System.IO;
 using System.Linq;
 using System.Runtime.Caching;
+using System.Runtime.CompilerServices;
 using System.Text;
 using System.Threading.Tasks;
 using System.Xml;
@@ -77,7 +78,8 @@ public enum CollectionType
         SchemaSnapshot,
         IdentityColumns,
         Instance,
-        RunningJobs
+        RunningJobs,
+        TableSize
     }
 
     public enum HostPlatform
@@ -95,7 +97,7 @@ public class DBCollector
         private DataTable dtInternalPerfCounters;
         public int PerformanceCollectionPeriodMins = 60;
         private string computerName;
-        private readonly CollectionType[] azureCollectionTypes = new[] { CollectionType.SlowQueries, CollectionType.AzureDBElasticPoolResourceStats, CollectionType.AzureDBServiceObjectives, CollectionType.AzureDBResourceStats, CollectionType.CPU, CollectionType.DBFiles, CollectionType.Databases, CollectionType.DBConfig, CollectionType.TraceFlags, CollectionType.ObjectExecutionStats, CollectionType.BlockingSnapshot, CollectionType.IOStats, CollectionType.Waits, CollectionType.ServerProperties, CollectionType.DBTuningOptions, CollectionType.SysConfig, CollectionType.DatabasePrincipals, CollectionType.DatabaseRoleMembers, CollectionType.DatabasePermissions, CollectionType.OSInfo, CollectionType.CustomChecks, CollectionType.PerformanceCounters, CollectionType.VLF, CollectionType.DatabaseQueryStoreOptions, CollectionType.AzureDBResourceGovernance, CollectionType.RunningQueries, CollectionType.IdentityColumns };
+        private readonly CollectionType[] azureCollectionTypes = new[] { CollectionType.SlowQueries, CollectionType.AzureDBElasticPoolResourceStats, CollectionType.AzureDBServiceObjectives, CollectionType.AzureDBResourceStats, CollectionType.CPU, CollectionType.DBFiles, CollectionType.Databases, CollectionType.DBConfig, CollectionType.TraceFlags, CollectionType.ObjectExecutionStats, CollectionType.BlockingSnapshot, CollectionType.IOStats, CollectionType.Waits, CollectionType.ServerProperties, CollectionType.DBTuningOptions, CollectionType.SysConfig, CollectionType.DatabasePrincipals, CollectionType.DatabaseRoleMembers, CollectionType.DatabasePermissions, CollectionType.OSInfo, CollectionType.CustomChecks, CollectionType.PerformanceCounters, CollectionType.VLF, CollectionType.DatabaseQueryStoreOptions, CollectionType.AzureDBResourceGovernance, CollectionType.RunningQueries, CollectionType.IdentityColumns, CollectionType.TableSize };
         private readonly CollectionType[] azureOnlyCollectionTypes = new[] { CollectionType.AzureDBElasticPoolResourceStats, CollectionType.AzureDBResourceStats, CollectionType.AzureDBServiceObjectives, CollectionType.AzureDBResourceGovernance };
         private readonly CollectionType[] azureMasterOnlyCollectionTypes = new[] { CollectionType.AzureDBElasticPoolResourceStats };
         public DBADashSource Source;
@@ -132,6 +134,11 @@ public class DBCollector
 
         private bool IsTableValuedConstructorsSupported => SQLVersion.Major > 9;
 
+        private const int TableSizeCollectionThresholdMBDefault = 100;
+        private const string TableSizeDatabasesDefault = "*";
+        private const int TableSizeMaxTableThresholdDefault = 2000;
+        private const int TableSizeMaxDatabaseThreshold = 500;
+
         public int Job_instance_id
         {
             get
@@ -253,7 +260,7 @@ public static bool ShouldRetry(Exception ex)
         {
             if (ex is SqlException sqlEx)
             {
-                return !ExcludedErrorCodes.Contains(sqlEx.Number);
+                return !ExcludedErrorCodes.Contains(sqlEx.Number) && sqlEx.Message != "Max databases exceeded for Table Size collection";
             }
             return true;
         }
@@ -527,6 +534,10 @@ private bool CollectionTypeIsApplicable(CollectionType collectionType)
             {
                 return false; // Required & collected by default
             }
+            else if (collectionType == CollectionType.TableSize && SQLVersion.Major <= 12 && !IsAzureDB)
+            {
+                return false; // Table size collection not supported on SQL 2014 and below
+            }
             else
             {
                 return true;
@@ -545,13 +556,13 @@ public void Collect(CollectionType collectionType)
             try
             {
                 retryPolicy.Execute(
-                  _ =>
-                  {
-                      StartCollection(collectionType.ToString());
-                      ExecuteCollection(collectionType);
-                      StopCollection();
-                  },
-                  new Context(collectionTypeString)
+                    _ =>
+                    {
+                        StartCollection(collectionType.ToString());
+                        ExecuteCollection(collectionType);
+                        StopCollection();
+                    },
+                    new Context(collectionTypeString)
                 );
             }
             catch (Exception ex)
@@ -914,6 +925,14 @@ private void ExecuteCollection(CollectionType collectionType)
             {
                 param = new[] { new SqlParameter("IOCollectionLevel", Source.IOCollectionLevel) };
             }
+            else if (collectionType == CollectionType.TableSize)
+            {
+                param = new[] { new SqlParameter("SizeThresholdMB", Source.TableSizeCollectionThresholdMB ?? TableSizeCollectionThresholdMBDefault),
+                    new SqlParameter("TableSizeDatabases", Source.TableSizeDatabases ?? TableSizeDatabasesDefault),
+                    new SqlParameter("MaxTables", Source.TableSizeMaxTableThreshold ?? TableSizeMaxTableThresholdDefault ),
+                    new SqlParameter("MaxDatabases", Source.TableSizeMaxDatabaseThreshold ?? TableSizeMaxDatabaseThreshold)
+                };
+            }
 
             if (collectionType == CollectionType.Drives)
             {

DBADash/DBImporter.cs

@@ -295,7 +295,7 @@ private void UpgradeDS()
             "Jobs", "JobHistory", "AvailabilityReplicas", "AvailabilityGroups", "JobSteps",
             "DatabaseQueryStoreOptions", "ResourceGovernorConfiguration", "AzureDBResourceGovernance",
             "RunningQueries", "QueryText", "QueryPlans", "InternalPerformanceCounters", "MemoryUsage",
-            "SessionWaits", "IdentityColumns", "RunningJobs"
+            "SessionWaits", "IdentityColumns", "RunningJobs", "TableSize"
         };
 
         public void Update()

DBADash/SQL/SQLTableSize.sql

@@ -0,0 +1,119 @@
+/*
+DECLARE @SizeThresholdMB INT = 100 /* Table is excluded if the size is under this threshold */
+DECLARE @TableSizeDatabases NVARCHAR(MAX)='*' /* Comma separated list of databases to include. - character can be used to exclude */
+DECLARE @MaxDatabases INT = 500 /* If server has a large number of databases, fail the collection */
+DECLARE @MaxTables INT = 2000 /* Skip databases that have a very large number of tables */
+*/
+SET NOCOUNT ON
+DECLARE @SizeThresholdPages INT = @SizeThresholdMB * 128
+CREATE TABLE #tablesize ( 
+		[DB] nvarchar(128) NOT NULL, 
+		[database_id] smallint NOT NULL, 
+		[schema_name] nvarchar(128) NOT NULL, 
+		[object_name] nvarchar(128) NOT NULL, 
+		[object_id] int NOT NULL, 
+		[type] char(2) NOT NULL, 
+		[row_count] bigint NOT NULL, 
+		[reserved_pages] bigint, 
+		[used_pages] bigint, 
+		[data_pages] bigint 
+)
+CREATE TABLE #Databases(
+	name NVARCHAR(128),
+
+)
+
+INSERT INTO #Databases(name)
+SELECT D.name
+FROM sys.databases D
+WHERE D.state  = 0
+AND HAS_DBACCESS(D.name)=1
+AND D.database_id <> 2
+AND EXISTS(
+		SELECT 1 
+		FROM STRING_SPLIT(@TableSizeDatabases,',') SS
+		WHERE (TRIM(SS.value) = D.name OR SS.value ='*')
+)
+AND NOT EXISTS(
+			SELECT 1 
+			FROM STRING_SPLIT(@TableSizeDatabases,',') SS
+			WHERE STUFF(TRIM(SS.value),1,1,'') = D.name 
+			AND TRIM(SS.value) LIKE '-%'
+	)
+
+IF @@ROWCOUNT > @MaxDatabases
+BEGIN
+	RAISERROR('Max databases exceeded for Table Size collection',11,1)
+	RETURN
+END
+
+DECLARE @SQL NVARCHAR(MAX)
+DECLARE @DBName SYSNAME
+DECLARE DBs CURSOR FAST_FORWARD READ_ONLY LOCAL FOR
+	SELECT D.name
+	FROM #Databases D
+
+OPEN DBs
+FETCH NEXT FROM DBs INTO @DBName
+
+WHILE @@FETCH_STATUS = 0
+BEGIN
+	SET @SQL =  N'USE ' + QUOTENAME(@DBName)  + '
+	IF (SELECT COUNT(*) 
+		FROM (
+				/* 
+					Using TOP to avoid running expensive has_access check more often then needed.  
+					Useful if there is a very large number of tables and the user isn''t db_owner
+				*/
+				SELECT TOP(@MaxTables+1) object_id 
+				FROM sys.objects 
+				WHERE type = ''U''
+			 ) T
+		) > @MaxTables
+	BEGIN
+		SELECT	DB_NAME() AS DB,
+			DB_ID() as database_id,
+			''{DBADashError}'' as schema_name,
+			''{TableCountExceededThreshold}'' as object_name,
+			-1 AS object_id,
+			''E'' AS type,
+			-1 AS row_count,
+			-1 AS reserved_pages,
+			-1 AS used_pages,
+			-1 AS data_pages
+		RETURN
+	END
+	SELECT	DB_NAME() AS DB,
+			DB_ID() as database_id,
+			s.name as schema_name,
+			so.name as object_name,
+			ps.object_id,
+			so.type,
+			SUM (CASE WHEN (ps.index_id < 2) THEN row_count ELSE 0 END) AS row_count,
+			SUM (ps.reserved_page_count) AS reserved_pages,
+			SUM (ps.used_page_count) AS used_pages,
+			SUM (CASE WHEN (ps.index_id < 2) THEN (ps.in_row_data_page_count + ps.lob_used_page_count + ps.row_overflow_used_page_count)
+						ELSE ps.lob_used_page_count + ps.row_overflow_used_page_count
+						END
+					) AS data_pages
+	FROM sys.dm_db_partition_stats ps
+	INNER JOIN sys.objects so ON ps.object_id = so.object_id
+	INNER JOIN sys.schemas s ON so.schema_id = s.schema_id
+	WHERE so.type = ''U''
+	GROUP BY ps.object_id,so.name,so.type_desc,s.name,so.type
+	HAVING SUM (ps.used_page_count) > @SizeThresholdPages'
+
+
+	IF HAS_DBACCESS(@DBName)=1
+	BEGIN
+		INSERT INTO #tablesize ([DB], [database_id], [schema_name], [object_name], [object_id], [type], [row_count], [reserved_pages], [used_pages], [data_pages])
+		EXEC sp_executesql  @SQL,N'@SizeThresholdPages INT,@MaxTables INT',@SizeThresholdPages,@MaxTables
+	END
+
+	FETCH NEXT FROM DBs INTO @DBName
+END
+CLOSE DBs
+DEALLOCATE DBs
+
+SELECT SYSUTCDATETIME() AS SnapshotDate,*
+FROM #tablesize

DBADashDB/DBADashDB.sqlproj

@@ -713,6 +713,13 @@
     <Build Include="Security\UserData.sql" />
     <Build Include="dbo\Views\PartitionHelper.sql" />
     <Build Include="dbo\Views\PartitionConfiguration.sql" />
+    <Build Include="dbo\Tables\TableSize.sql" />
+    <Build Include="dbo\Stored Procedures\TableSize_Upd.sql" />
+    <Build Include="dbo\User Defined Types\TableSize.sql" />
+    <Build Include="dbo\Stored Procedures\TableSize_Get.sql" />
+    <Build Include="dbo\Stored Procedures\TableSizeHistory_Get.sql" />
+    <Build Include="Storage\PF_TableSize.sql" />
+    <Build Include="Storage\TableSize.sql" />
   </ItemGroup>
   <ItemGroup>
     <PostDeploy Include="Script.PostDeployment1.sql" />

DBADashDB/Script.PostDeployment1.sql

@@ -1140,9 +1140,13 @@ FROM (VALUES('ObjectExecutionStats',120),
 				('CollectionErrorLog',14),
 				('MemoryUsage',30),
 				('SessionWaits',30),
-				('IdentityColumnsHistory',730)
+				('IdentityColumnsHistory',730),
+				('TableSize',730)
 				) AS t(TableName,RetentionDays)
-WHERE NOT EXISTS(SELECT 1 FROM dbo.DataRetention DR WHERE DR.TableName = T.TableName)
+WHERE NOT EXISTS(SELECT 1 
+				FROM dbo.DataRetention DR
+				WHERE DR.TableName = T.TableName
+				AND DR.SchemaName = 'dbo')
 
 DELETE dbo.OSLoadedModulesStatus
 WHERE IsSystem=1
@@ -1366,7 +1370,8 @@ FROM
 (-1,'ResourceGovernorConfiguration',1445,2880),
 (-1,'MemoryUsage',5,10),
 (-1,'IdentityColumns',10080,20160),
-(-1,'RunningJobs',5,10)
+(-1,'RunningJobs',5,10),
+(-1,'TableSize',4320,11520)
 ) T(InstanceID,Reference,WarningThreshold,CriticalThreshold)
 WHERE NOT EXISTS(SELECT 1 FROM dbo.CollectionDatesThresholds CDT WHERE CDT.InstanceID = T.InstanceID AND CDT.Reference = T.Reference)
 

DBADashDB/Storage/PF_TableSize.sql

@@ -0,0 +1 @@
+CREATE PARTITION FUNCTION [PF_TableSize](DATETIME2) AS RANGE RIGHT FOR VALUES()

DBADashDB/Storage/TableSize.sql

@@ -0,0 +1 @@
+CREATE PARTITION SCHEME PS_TableSize AS PARTITION PF_TableSize ALL TO([PRIMARY])

DBADashDB/dbo/Stored Procedures/Summary_Upd.sql

@@ -86,6 +86,7 @@ WITH err AS (
 	FROM dbo.CollectionErrorLog E
 	WHERE ErrorDate>=@ErrorsFrom
 	AND ErrorContext NOT LIKE '%[[]Retrying]'
+	AND NOT (ErrorSource='TableSize' AND ErrorMessage LIKE 'Warning%')
 	GROUP BY InstanceID,ErrorSource
 )
 SELECT err.InstanceID,