Add option to configure IO collection level

IO Collection level grouped on collection.
Note: This has issues if log shipping is used.  See. #594

DBADash.Test/PowerShellScriptsTest.cs

@@ -3,10 +3,12 @@
 using System;
 using System.Diagnostics;
 using System.IO;
+using DBADash;
+using Microsoft.SqlServer.Management.HadrModel;
 using static DBADashConfig.Test.Helper;
+
 namespace DBADashConfig.Test
 {
-
     [TestClass]
     public class PowerShellScriptsTest
     {
@@ -22,13 +24,13 @@ public static void ClassCleanup()
             Console.WriteLine("Inside ClassCleanup");
         }
 
-
         [TestMethod]
         public void SetDBADashDestinationTest()
         {
             var builder = GetRandomConnectionString();
             string dest = builder.ToString();
-            var psi = new ProcessStartInfo("powershell", String.Format("-File \"Set-DBADashDestination.ps1\" -ConnectionString \"{0}\" -SkipValidation", dest));
+            var psi = new ProcessStartInfo("powershell",
+                $"-File \"Set-DBADashDestination.ps1\" -ConnectionString \"{dest}\" -SkipValidation");
             Helper.RunProcess(psi);
 
             string json = Helper.GetConfigJson();
@@ -42,18 +44,18 @@ public void SetDBADashDestinationTest()
             Assert.IsFalse(json.Contains(builder.Password), "Plain text password found in json");
         }
 
-
         [TestMethod]
-        [DataRow(-1, false, int.MaxValue, Int32.MaxValue, Int32.MaxValue, Int32.MaxValue, "")]
-        [DataRow(999, true, 2, 1000, 1001, 8000, "*")]
-        [DataRow(2000, true, 1, 2000, 2001, 6400, "DB1,DB2,DB3")]
-        public void AddDBADashSourceTest(int slowQueryThreshold, bool planCollectionEnabled, int planCollectionCountThreshold, int planCollectionCPUThreshold, int planCollectionDurationThreshold, int PlanCollectionMemoryGrantThreshold, string schemaSnapshotDBs)
+        [DataRow(-1, false, int.MaxValue, Int32.MaxValue, Int32.MaxValue, Int32.MaxValue, "", 1)]
+        [DataRow(999, true, 2, 1000, 1001, 8000, "*", 2)]
+        [DataRow(2000, true, 1, 2000, 2001, 6400, "DB1,DB2,DB3", 3)]
+        public void AddDBADashSourceTest(int slowQueryThreshold, bool planCollectionEnabled, int planCollectionCountThreshold, int planCollectionCPUThreshold, int planCollectionDurationThreshold, int PlanCollectionMemoryGrantThreshold, string schemaSnapshotDBs, DBADashSource.IOCollectionLevels IOCollectionLevel)
         {
             // Add connection
             var cnt = Helper.GetConnectionCount();
             var builder = GetRandomConnectionString();
             string source = builder.ToString();
-            var psi = new ProcessStartInfo("powershell", String.Format("-File \"Add-DBADashSource.ps1\" -ConnectionString \"{0}\" --SkipValidation", source));
+            var psi = new ProcessStartInfo("powershell",
+                $"-File \"Add-DBADashSource.ps1\" -ConnectionString \"{source}\" --SkipValidation");
             RunProcess(psi);
 
             // Read the new config from disk
@@ -65,26 +67,28 @@ public void AddDBADashSourceTest(int slowQueryThreshold, bool planCollectionEnab
             // Convert config to CollectionConfig object for additional checks
             var cfg = DBADash.CollectionConfig.Deserialize(json);
             var newCnt = cfg.SourceConnections.Count;
-            Assert.IsTrue(newCnt == cnt + 1, string.Format("Expected {0} source connections instead of {1}", cnt + 1, newCnt));
+            Assert.IsTrue(newCnt == cnt + 1, $"Expected {cnt + 1} source connections instead of {newCnt}");
             var src = cfg.GetSourceFromConnectionString(builder.ConnectionString);
             Assert.IsTrue(src.NoWMI == false, "source expected NoWMI=false");
 
             // Update the connection and validate it's been updated
             builder.Password = Guid.NewGuid().ToString();
-            psi = new ProcessStartInfo("powershell", String.Format("-File \"Add-DBADashSource.ps1\" -ConnectionString \"{0}\" -SkipValidation --Replace -NoWMI -SlowQueryThresholdMs {1}", source, slowQueryThreshold));
+            psi = new ProcessStartInfo("powershell",
+                $"-File \"Add-DBADashSource.ps1\" -ConnectionString \"{source}\" -SkipValidation --Replace -NoWMI -SlowQueryThresholdMs {slowQueryThreshold}");
             if (!string.IsNullOrEmpty(schemaSnapshotDBs))
             {
-                psi.Arguments += String.Format(" -SchemaSnapshotDBs \"{0}\"", schemaSnapshotDBs);
+                psi.Arguments += $" -SchemaSnapshotDBs \"{schemaSnapshotDBs}\"";
             }
             if (planCollectionEnabled)
             {
                 psi.Arguments += " -PlanCollectionEnabled";
-                psi.Arguments += string.Format(" -PlanCollectionCountThreshold {0}", planCollectionCountThreshold);
-                psi.Arguments += string.Format(" -PlanCollectionCPUThreshold {0}", planCollectionCPUThreshold);
-                psi.Arguments += string.Format(" -PlanCollectionDurationThreshold {0}", planCollectionDurationThreshold);
-                psi.Arguments += string.Format(" -PlanCollectionMemoryGrantThreshold {0}", PlanCollectionMemoryGrantThreshold);
+                psi.Arguments += $" -PlanCollectionCountThreshold {planCollectionCountThreshold}";
+                psi.Arguments += $" -PlanCollectionCPUThreshold {planCollectionCPUThreshold}";
+                psi.Arguments += $" -PlanCollectionDurationThreshold {planCollectionDurationThreshold}";
+                psi.Arguments += $" -PlanCollectionMemoryGrantThreshold {PlanCollectionMemoryGrantThreshold}";
             }
 
+            psi.Arguments += $" -IOCollectionLevel {(int)IOCollectionLevel}";
             RunProcess(psi);
 
             // Read config from disk again
@@ -94,7 +98,7 @@ public void AddDBADashSourceTest(int slowQueryThreshold, bool planCollectionEnab
 
             newCnt = cfg.SourceConnections.Count;
             // This time we should be replacing the existing connection - so the count will be the same
-            Assert.IsTrue(newCnt == cnt + 1, string.Format("Expected {0} source connections instead of {1}", cnt + 1, newCnt));
+            Assert.IsTrue(newCnt == cnt + 1, $"Expected {cnt + 1} source connections instead of {newCnt}");
             src = cfg.GetSourceFromConnectionString(builder.ConnectionString);
             Console.WriteLine(src.SlowQueryThresholdMs);
             // Check updates worked
@@ -104,6 +108,7 @@ public void AddDBADashSourceTest(int slowQueryThreshold, bool planCollectionEnab
             Assert.IsTrue(src.PlanCollectionCountThreshold == planCollectionCountThreshold, "PlanCollectionCountThreshold Test");
             Assert.IsTrue(src.PlanCollectionCPUThreshold == planCollectionCPUThreshold, "PlanCollectionCountThreshold Test");
             Assert.IsTrue(src.PlanCollectionMemoryGrantThreshold == PlanCollectionMemoryGrantThreshold, "PlanCollectionMemoryGrantThreshold Test");
+            Assert.IsTrue(src.IOCollectionLevel == IOCollectionLevel, "IOCollectionLevel Test");
             if (string.IsNullOrEmpty(schemaSnapshotDBs))
             {
                 Assert.IsTrue(string.IsNullOrEmpty(src.SchemaSnapshotDBs), "SchemaSnaspshotDBs Test");
@@ -113,8 +118,5 @@ public void AddDBADashSourceTest(int slowQueryThreshold, bool planCollectionEnab
                 Assert.IsTrue(src.SchemaSnapshotDBs == schemaSnapshotDBs, "SchemaSnaspshotDBs Test");
             }
         }
-
     }
-
-
 }

DBADash/DBADashSource.cs

@@ -2,14 +2,23 @@
 using Newtonsoft.Json;
 using System;
 using System.ComponentModel;
+using Microsoft.SqlServer.Dac.Model;
 using static DBADash.DBADashConnection;
 
 namespace DBADash
 {
     public class DBADashSource
     {
-        private Int32 slowQueryThresholdMs = -1;
+        public enum IOCollectionLevels
+        {
+            Full = 1,
+            InstanceOnly = 2,
+            Drive = 3,
+            Database = 4,
+            DriveAndDatabase = 5
+        }
 
+        private Int32 slowQueryThresholdMs = -1;
         private CollectionSchedules collectionSchedules;
         private PlanCollectionThreshold runningQueryPlanThreshold;
         private string schemaSnapshotDBs;
@@ -22,6 +31,8 @@ public class DBADashSource
         public string ConnectionID { get; set; }
         public bool ScriptAgentJobs { get; set; } = true;
 
+        public IOCollectionLevels IOCollectionLevel { get; set; } = IOCollectionLevels.Full;
+
         public CollectionSchedules CollectionSchedules
         {
             get => SourceConnection != null && SourceConnection.Type == ConnectionType.SQL ? collectionSchedules : null;

DBADash/DBCollector.cs

@@ -516,7 +516,7 @@ private static string ByteArrayToHexString(byte[] bytes)
         private DataTable GetPlans(string plansSQL)
         {
             if (string.IsNullOrEmpty(plansSQL)) return null;
-     
+
             using var cn = new SqlConnection(ConnectionString);
             using var da = new SqlDataAdapter(plansSQL, cn);
             var dt = new DataTable("QueryPlans");
@@ -837,6 +837,10 @@ private void ExecuteCollection(CollectionType collectionType)
             {
                 param = new[] { new SqlParameter("IdentityCollectionThreshold", IdentityCollectionThreshold) };
             }
+            else if (collectionType == CollectionType.IOStats)
+            {
+                param = new[] { new SqlParameter("IOCollectionLevel", Source.IOCollectionLevel) };
+            }
 
             if (collectionType == CollectionType.Drives)
             {
@@ -966,7 +970,7 @@ private DataSet GetPerformanceCounters()
             return ds;
         }
 
-        private void MergeCustomPerformanceCounters(DataTable dt,DataTable userDT)
+        private void MergeCustomPerformanceCounters(DataTable dt, DataTable userDT)
         {
             if (dt.Columns.Count == userDT.Columns.Count)
             {
@@ -1006,7 +1010,7 @@ private void CollectPerformanceCounters()
             if (ds.Tables.Count == 2)
             {
                 var userDT = ds.Tables[1];
-                MergeCustomPerformanceCounters(dt,userDT);
+                MergeCustomPerformanceCounters(dt, userDT);
             }
             ds.Tables.Remove(dt);
             dt.TableName = "PerformanceCounters";
@@ -1310,7 +1314,7 @@ private void CollectDriversWMI()
                         "ClassGuid" => typeof(Guid),
                         _ => typeof(string)
                     };
-                   
+
                     dtDrivers.Columns.Add(p, columnType);
                 }
 
@@ -1421,7 +1425,7 @@ private string PVDriverVersion()
         private void CollectDrivesWMI()
         {
             if (Data.Tables.Contains("Drives") || !OperatingSystem.IsWindows()) return;
-            DataTable drives = new ("Drives")
+            DataTable drives = new("Drives")
             {
                 Columns =
                 {

DBADash/SQL/SQLIOStats.sql

@@ -1,32 +1,73 @@
-IF SERVERPROPERTY('EngineEdition')=5 -- Azure SQL Database
+/*  
+	DECLARE @IOCollectionLevel INT       
+	Full = 1,
+	InstanceOnly = 2,
+	Drive = 3,
+	Database = 4,
+	DriveAndDatabase = 5
+*/
+IF SERVERPROPERTY('EngineEdition')=5 -- Azure SQL Database
 BEGIN
 	SELECT GETUTCDATE() AS SnapshotDate,
-		   [database_id],
-		   [file_id],
-		   [sample_ms],
-		   [num_of_reads],
-		   [num_of_bytes_read],
-		   [io_stall_read_ms],
-		   [num_of_writes],
-		   [num_of_bytes_written],
-		   [io_stall_write_ms],
-		   [io_stall],
-		   [size_on_disk_bytes]
-	FROM sys.dm_io_virtual_file_stats(DB_ID(), NULL) vfs;
+		   vfs.database_id,
+		   vfs.file_id,
+		   vfs.sample_ms,
+		   vfs.num_of_reads,
+		   vfs.num_of_bytes_read,
+		   vfs.io_stall_read_ms,
+		   vfs.num_of_writes,
+		   vfs.num_of_bytes_written,
+		   vfs.io_stall_write_ms,
+		   vfs.io_stall,
+		   vfs.size_on_disk_bytes,
+		   CAST('-' AS CHAR(1)) AS drive
+	FROM sys.dm_io_virtual_file_stats(DB_ID(), NULL) vfs
 END
 ELSE
 BEGIN
-	SELECT GETUTCDATE() AS SnapshotDate,
-		   [database_id],
-		   [file_id],
-		   [sample_ms],
-		   [num_of_reads],
-		   [num_of_bytes_read],
-		   [io_stall_read_ms],
-		   [num_of_writes],
-		   [num_of_bytes_written],
-		   [io_stall_write_ms],
-		   [io_stall],
-		   [size_on_disk_bytes]
-	FROM sys.dm_io_virtual_file_stats(NULL, NULL) vfs;
+	IF @IOCollectionLevel = 1 /* File Level - no aggregation */
+	BEGIN
+		SELECT GETUTCDATE() AS SnapshotDate,
+			   vfs.database_id,
+			   vfs.file_id,
+			   vfs.sample_ms,
+			   vfs.num_of_reads,
+			   vfs.num_of_bytes_read,
+			   vfs.io_stall_read_ms,
+			   vfs.num_of_writes,
+			   vfs.num_of_bytes_written,
+			   vfs.io_stall_write_ms,
+			   vfs.io_stall,
+			   vfs.size_on_disk_bytes,
+			   CAST(LEFT(mf.physical_name,1) AS CHAR(1)) AS drive
+		FROM sys.dm_io_virtual_file_stats(NULL, NULL) vfs
+		JOIN sys.master_files mf on vfs.database_id = mf.database_id AND vfs.file_id = mf.file_id;
+	END
+	ELSE IF @IOCollectionLevel IN(2,3,4,5)
+	BEGIN
+		SELECT GETUTCDATE() AS SnapshotDate,
+				CASE WHEN @IOCollectionLevel IN(4,5) THEN mf.database_id ELSE -1 END AS database_id, 
+				-1 AS file_id,
+				MAX(vfs.sample_ms) AS sample_ms,
+				SUM(vfs.num_of_reads) AS num_of_reads,
+				SUM(vfs.num_of_bytes_read) AS num_of_bytes_read,
+				SUM(vfs.io_stall_read_ms) AS io_stall_read_ms,
+				SUM(vfs.num_of_writes) AS num_of_writes,
+				SUM(vfs.num_of_bytes_written) AS num_of_bytes_written,
+				SUM(vfs.io_stall_write_ms) AS io_stall_write_ms,
+				SUM(vfs.io_stall) AS io_stall,
+				SUM(vfs.size_on_disk_bytes) AS size_on_disk_bytes,
+				CASE WHEN @IOCollectionLevel IN(3,5) THEN CAST(LEFT(mf.physical_name,1) AS CHAR(1)) ELSE '-' END AS drive
+		FROM sys.dm_io_virtual_file_stats(NULL, NULL) vfs
+		JOIN sys.master_files mf on vfs.database_id = mf.database_id AND vfs.file_id = mf.file_id
+		GROUP BY CASE WHEN @IOCollectionLevel IN(3,5) THEN CAST(LEFT(mf.physical_name,1) AS CHAR(1)) ELSE '-' END,
+				CASE WHEN @IOCollectionLevel IN(4,5) THEN mf.database_id ELSE -1 END
+
+	END
+	ELSE
+	BEGIN
+		DECLARE @Msg VARCHAR(1000)
+		SET @Msg = 'Invalid @IOCollectionLevel: ' + CAST(@IOCollectionLevel AS VARCHAR(10))
+		RAISERROR(@Msg,11,1)
+	END
 END

DBADashConfig/Add-DBADashSource.ps1

@@ -28,6 +28,8 @@
     Keep a copy of the old config file before saving
 .PARAMETER Replace
     Option to replace the existing connection if it already exists
+.PARAMETER IOCollectionLevel
+    1=Full,2=Drive,3=Instance
 .EXAMPLE
     ./Add-DBADashSource -ConnectionString "Data Source=MYSERVER;Integrated Security=True;Encrypt=True;Trust Server Certificate=True"
 .EXAMPLE
@@ -50,7 +52,8 @@ Param(
     [bool]$BackupConfig=$true,
     [switch]$Replace,
     [switch]$RestartService,
-    [bool]$CollectSessionWaits=$true
+    [bool]$CollectSessionWaits=$true,
+    [int]$IOCollectionLevel=1
 )
 if(!$SchemaSnapshotDBs){
     $SchemaSnapshotDBs="<null>"
@@ -65,6 +68,7 @@ $command+="--PlanCollectionCPUThreshold $PlanCollectionCPUThreshold "
 $command+="--PlanCollectionDurationThreshold $PlanCollectionDurationThreshold "
 $command+="--PlanCollectionMemoryGrantThreshold $PlanCollectionMemoryGrantThreshold "
 $command+="--SchemaSnapshotDBs `"$SchemaSnapshotDBs`" "
+$command+="--IOCollectionLevel $IOCollectionLevel "
 if ($PlanCollectionEnabled.IsPresent){
     $command+="--PlanCollectionEnabled "
 }   

DBADashConfig/Options.cs

@@ -1,4 +1,5 @@
 using CommandLine;
+using DBADash;
 
 namespace DBADashConfig
 {
@@ -58,6 +59,10 @@ public class Options
         [Option("ServiceName", Default = "", Required = false, HelpText = "Use with -a SetServiceName to set the service name for the DBA Dash service.")]
         public string ServiceName { get; set; } = "";
 
+        [Option("IOCollectionLevel", Default = 1, Required = false,
+            HelpText = "IO Collection Level.  1=Full, 2=Drive, 3=Instance")]
+        public int IOCollectionLevel { get; set; } = 1;
+
         public enum CommandLineActionOption
         {
             Add,
@@ -72,6 +77,4 @@ public enum CommandLineActionOption
             SetServiceName
         }
     }
-
-
-}
+}