Skip to content

Commit

Permalink
Add option to Hard Delete instance
Browse files Browse the repository at this point in the history
Remove legacy BlockingSnapshot table.  Don't need to handle deletes for this table - should be removed instead.
Add SPs to remove data for a specific instance from various tables.  Can also be used to reduce retention for a specific instance (Not efficient compared to partition switching though)
Add data cleanup for BlockingSnapshotSummary table based on retention settings for RunningQueries.
Add indexes to help support delete operation
Remove certain FKs to avoid the need for a supporting index. #119
Update Docs
  • Loading branch information
DavidWiseman committed Jun 27, 2022
1 parent ec713c7 commit 6591811
Show file tree
Hide file tree
Showing 49 changed files with 1,269 additions and 370 deletions.
33 changes: 26 additions & 7 deletions DBADashDB/DBADashDB.sqlproj
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
<AllowSnapshotIsolation>True</AllowSnapshotIsolation>
<CompatibilityMode>130</CompatibilityMode>
<TargetFrameworkProfile />
<AutoUpdateStatisticsAsynchronously>True</AutoUpdateStatisticsAsynchronously>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<BuildScriptName>$(MSBuildProjectName).sql</BuildScriptName>
Expand Down Expand Up @@ -219,10 +220,6 @@
<Build Include="Report\Stored Procedures\AWSDriver_Get.sql" />
<Build Include="Report\Stored Procedures\Drivers_Get.sql" />
<Build Include="dbo\Tables\BlockingSnapshotSummary.sql" />
<Build Include="dbo\Tables\BlockingSnapshot.sql" />
<Build Include="dbo\Stored Procedures\BlockingSnapshot_Upd.sql" />
<Build Include="dbo\User Defined Types\BlockingSnapshot.sql" />
<Build Include="dbo\Functions\BlockingSnapshotRecursiveStats.sql" />
<Build Include="Report\Stored Procedures\BlockingSummary.sql" />
<Build Include="Report\Stored Procedures\Blocking.sql" />
<Build Include="Staging\Tables\IOStats.sql" />
Expand Down Expand Up @@ -254,14 +251,11 @@
<Build Include="Security\Switch.sql" />
<Build Include="Switch\Tables\Waits.sql" />
<Build Include="Switch\Tables\CPU.sql" />
<Build Include="Switch\Tables\BlockingSnapshot.sql" />
<Build Include="dbo\Tables\DataRetention.sql" />
<Build Include="Storage\PF_Waits.sql" />
<Build Include="Storage\PF_CPU.sql" />
<Build Include="Storage\PF_BlockingSnapshot.sql" />
<Build Include="Storage\PS_Waits.sql" />
<Build Include="Storage\PS_CPU.sql" />
<Build Include="Storage\PS_BlockingSnapshot.sql" />
<Build Include="dbo\Stored Procedures\PurgeData.sql" />
<Build Include="dbo\Tables\OSLoadedModules.sql" />
<Build Include="dbo\Tables\DBTuningOptionsHistory.sql" />
Expand Down Expand Up @@ -648,6 +642,31 @@
<Build Include="dbo\Stored Procedures\InstanceAlias_Upd.sql" />
<Build Include="dbo\Stored Procedures\CounterThresholds_Upd.sql" />
<Build Include="dbo\Stored Procedures\CounterThresholds_Get.sql" />
<Build Include="dbo\Stored Procedures\AzureDBElasticPoolResourceStats_Del.sql" />
<Build Include="dbo\Stored Procedures\AzureDBElasticPoolResourceStats_60MIN_Del.sql" />
<Build Include="dbo\Stored Procedures\AzureDBResourceStats_60MIN_Del.sql" />
<Build Include="dbo\Stored Procedures\AzureDBResourceStats_Del.sql" />
<Build Include="dbo\Stored Procedures\CPU_Del.sql" />
<Build Include="dbo\Stored Procedures\CPU_60MIN_Del.sql" />
<Build Include="dbo\Stored Procedures\DBIOStats_Del.sql" />
<Build Include="dbo\Stored Procedures\DBIOStats_60MIN_Del.sql" />
<Build Include="dbo\Stored Procedures\JobStats_60MIN_Del.sql" />
<Build Include="dbo\Stored Procedures\MemoryUsage_Del.sql" />
<Build Include="dbo\Stored Procedures\ObjectExecutionStats_Del.sql" />
<Build Include="dbo\Stored Procedures\ObjectExecutionStats_60MIN_Del.sql" />
<Build Include="dbo\Stored Procedures\PerformanceCounters_Del.sql" />
<Build Include="dbo\Stored Procedures\PerformanceCounters_60MIN_Del.sql" />
<Build Include="dbo\Stored Procedures\RunningQueries_Del.sql" />
<Build Include="dbo\Stored Procedures\RunningQueriesSummary_Del.sql" />
<Build Include="dbo\Stored Procedures\SessionWaits_Del.sql" />
<Build Include="dbo\Stored Procedures\SlowQueries_Del.sql" />
<Build Include="dbo\Stored Procedures\SlowQueriesStats_Del.sql" />
<Build Include="dbo\Stored Procedures\Waits_Del.sql" />
<Build Include="dbo\Stored Procedures\Waits_60MIN_Del.sql" />
<Build Include="dbo\Stored Procedures\DBFileSnapshot_Del.sql" />
<Build Include="dbo\Stored Procedures\DriveSnapshot_Del.sql" />
<Build Include="dbo\Stored Procedures\PurgeBlockingSnapshotSummary.sql" />
<Build Include="dbo\Stored Procedures\BlockingSnapshotSummary_Del.sql" />
</ItemGroup>
<ItemGroup>
<PostDeploy Include="Script.PostDeployment1.sql" />
Expand Down
6 changes: 4 additions & 2 deletions DBADashDB/Script.PostDeployment1.sql
Original file line number Diff line number Diff line change
Expand Up @@ -1125,7 +1125,6 @@ FROM (VALUES('ObjectExecutionStats',120),
('Waits',120),
('DBIOStats',120),
('CPU',365),
('BlockingSnapshot',120),
('SlowQueries',120),
('AzureDBElasticPoolResourceStats',120),
('AzureDBResourceStats',120),
Expand Down Expand Up @@ -1588,7 +1587,9 @@ FROM (VALUES('PurgeCollectionErrorLog_StartDate'),
('PurgeQueryPlans_CompletedDate'),
('PurgePartitions_StartDate'),
('PurgePartitions_CompletedDate'),
('MemoryDumpAckDate')
('MemoryDumpAckDate'),
('PurgeBlockingSnapshotSummary_CompletedDate'),
('PurgeBlockingSnapshotSummary_StartDate')
) T(SettingName)
WHERE NOT EXISTS(SELECT 1
FROM dbo.Settings S
Expand Down Expand Up @@ -1799,3 +1800,4 @@ WHEN NOT MATCHED BY TARGET THEN
INSERT(object_name,counter_name,instance_name,SystemCriticalFrom,SystemCriticalTo,SystemWarningFrom,SystemWarningTo,SystemGoodFrom,SystemGoodTo)
VALUES(object_name,counter_name,instance_name,SystemCriticalFrom,SystemCriticalTo,SystemWarningFrom,SystemWarningTo,SystemGoodFrom,SystemGoodTo);

ALTER DATABASE [$(DatabaseName)] SET AUTO_UPDATE_STATISTICS_ASYNC ON WITH NO_WAIT
3 changes: 0 additions & 3 deletions DBADashDB/Storage/PF_BlockingSnapshot.sql

This file was deleted.

3 changes: 0 additions & 3 deletions DBADashDB/Storage/PS_BlockingSnapshot.sql

This file was deleted.

22 changes: 0 additions & 22 deletions DBADashDB/Switch/Tables/BlockingSnapshot.sql

This file was deleted.

3 changes: 1 addition & 2 deletions DBADashDB/Switch/Tables/ObjectExecutionStats_60MIN.sql
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
[execution_count] BIGINT NOT NULL,
[IsCompile] BIT NOT NULL,
[MaxExecutionsPerMin] DECIMAL (19, 6) NULL,
CONSTRAINT [PK_ObjectExecutionStats_60MIN] PRIMARY KEY CLUSTERED ([InstanceID] ASC, [SnapshotDate] ASC, [ObjectID] ASC) WITH (DATA_COMPRESSION = PAGE),
CONSTRAINT [FK_ObjectExecutionStats_60MIN_DBObjects] FOREIGN KEY ([ObjectID]) REFERENCES [dbo].[DBObjects] ([ObjectID])
CONSTRAINT [PK_ObjectExecutionStats_60MIN] PRIMARY KEY CLUSTERED ([InstanceID] ASC, [SnapshotDate] ASC, [ObjectID] ASC) WITH (DATA_COMPRESSION = PAGE)
);

28 changes: 0 additions & 28 deletions DBADashDB/dbo/Functions/BlockingSnapshotRecursiveStats.sql

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
CREATE PROC dbo.AzureDBElasticPoolResourceStats_60MIN_Del(
@InstanceID INT,
@DaysToKeep INT,
@BatchSize INT = 500000
)
AS
/*
Partitioning switching is used to efficiently remove data from this table for all instances.
This proc provides an alternative way to remove data associated with a specific instance.
*/
SET NOCOUNT ON
DECLARE @MaxDate DATETIME2
SELECT @MaxDate = CASE WHEN @DaysToKeep = 0 THEN '30000101' ELSE DATEADD(d,-@DaysToKeep,GETUTCDATE()) END

WHILE 1=1
BEGIN
DELETE EPRS
FROM dbo.AzureDBElasticPoolResourceStats_60MIN EPRS
WHERE EXISTS(SELECT 1
FROM dbo.AzureDBElasticPool EP
WHERE EP.InstanceID = @InstanceID
AND EP.PoolID = EPRS.PoolID
)
AND end_time < @MaxDate
OPTION(RECOMPILE)

IF @@ROWCOUNT < @BatchSize
BREAK
END
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
CREATE PROC dbo.AzureDBElasticPoolResourceStats_Del(
@InstanceID INT,
@DaysToKeep INT,
@BatchSize INT = 500000
)
AS
/*
Partitioning switching is used to efficiently remove data from this table for all instances.
This proc provides an alternative way to remove data associated with a specific instance.
*/
SET NOCOUNT ON
DECLARE @MaxDate DATETIME2
SELECT @MaxDate = CASE WHEN @DaysToKeep = 0 THEN '30000101' ELSE DATEADD(d,-@DaysToKeep,GETUTCDATE()) END

WHILE 1=1
BEGIN
DELETE EPRS
FROM dbo.AzureDBElasticPoolResourceStats EPRS
WHERE EXISTS(SELECT 1
FROM dbo.AzureDBElasticPool EP
WHERE EP.InstanceID = @InstanceID
AND EP.PoolID = EPRS.PoolID
)
AND end_time < @MaxDate
OPTION(RECOMPILE)

IF @@ROWCOUNT < @BatchSize
BREAK
END
25 changes: 25 additions & 0 deletions DBADashDB/dbo/Stored Procedures/AzureDBResourceStats_60MIN_Del.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
CREATE PROC dbo.AzureDBResourceStats_60MIN_Del(
@InstanceID INT,
@DaysToKeep INT,
@BatchSize INT=500000
)
AS
/*
Partitioning switching is used to efficiently remove data from this table for all instances.
This proc provides an alternative way to remove data associated with a specific instance.
*/
SET NOCOUNT ON
DECLARE @MaxDate DATETIME2(3)
SELECT @MaxDate = CASE WHEN @DaysToKeep = 0 THEN '30000101' ELSE DATEADD(d,-@DaysToKeep,GETUTCDATE()) END

WHILE 1=1
BEGIN
DELETE TOP(@BatchSize) EPRS
FROM dbo.AzureDBResourceStats_60MIN EPRS
WHERE InstanceID = @InstanceID
AND end_time < @MaxDate
OPTION(RECOMPILE)

IF @@ROWCOUNT < @BatchSize
BREAK
END
25 changes: 25 additions & 0 deletions DBADashDB/dbo/Stored Procedures/AzureDBResourceStats_Del.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
CREATE PROC dbo.AzureDBResourceStats_Del(
@InstanceID INT,
@DaysToKeep INT,
@BatchSize INT=500000
)
AS
/*
Partitioning switching is used to efficiently remove data from this table for all instances.
This proc provides an alternative way to remove data associated with a specific instance.
*/
SET NOCOUNT ON
DECLARE @MaxDate DATETIME2(3)
SELECT @MaxDate = CASE WHEN @DaysToKeep = 0 THEN '30000101' ELSE DATEADD(d,-@DaysToKeep,GETUTCDATE()) END

WHILE 1=1
BEGIN
DELETE TOP(@BatchSize) EPRS
FROM dbo.AzureDBResourceStats EPRS
WHERE InstanceID = @InstanceID
AND end_time < @MaxDate
OPTION(RECOMPILE)

IF @@ROWCOUNT < @BatchSize
BREAK
END
28 changes: 28 additions & 0 deletions DBADashDB/dbo/Stored Procedures/BlockingSnapshotSummary_Del.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
CREATE PROC dbo.BlockingSnapshotSummary_Del(
@InstanceID INT,
@DaysToKeep INT,
@BatchSize INT = 500000
)
AS
/*
Partitioning switching is used to efficiently remove data from this table for all instances.
This proc provides an alternative way to remove data associated with a specific instance.
*/
SET NOCOUNT ON
DECLARE @MaxDate DATETIME2
SELECT @MaxDate = CASE WHEN @DaysToKeep = 0 THEN '30000101' ELSE DATEADD(d,-@DaysToKeep,GETUTCDATE()) END

SELECT @MaxDate = DATEADD(d,-RetentionDays,GETUTCDATE())
FROM dbo.DataRetention
WHERE TableName = 'RunningQueries'

WHILE 1=1
BEGIN
DELETE TOP(@BatchSize)
FROM dbo.BlockingSnapshotSummary
WHERE SnapshotDateUTC < @MaxDate
AND InstanceID = @InstanceID

IF @@ROWCOUNT<@BatchSize
BREAK
END
Loading

0 comments on commit 6591811

Please sign in to comment.