Domain Table Storage

DNSHelper.psm1 - Allow for both specified DKIM selectors and MX lookup DomainAnalyser_All - Update to handle table row and return correct object - Process stored dkim selectors DomainAnalyser_GetTenantDomains - Rename from GetQueue DomainAnalyser_List - Migrate current cache files into table storage and delete - Return table storage results - Optimize the results output and reduce where-object use DomainAnalyser_Orchestrator - Import domain object to table row under TenantDetails property - Return Domain results for sending to durables ListDomainHealth - Update to store DKIM selectors when manually specified (editor/admin roles only) Scheduler_Orchestration & Standards_Orchestration - Add try/catch/finally to reduce errors in orchestrators
vincegremillion · Jun 20, 2022 · 8bb3b5f · 8bb3b5f
1 parent 1b5e5c9
commit 8bb3b5f
Show file tree

Hide file tree

Showing 11 changed files with 227 additions and 91 deletions.
diff --git a/DNSHelper.psm1 b/DNSHelper.psm1
@@ -1170,37 +1170,22 @@ function Read-DkimRecord {
     $ValidationWarns = [System.Collections.Generic.List[string]]::new()
     $ValidationFails = [System.Collections.Generic.List[string]]::new()
 
-    if (($Selectors | Measure-Object | Select-Object -ExpandProperty Count) -eq 0) {
-        # MX lookup, check for defined selectors
-        try {
-            $MXRecord = Read-MXRecord -Domain $Domain
-            foreach ($Selector in $MXRecord.Selectors) {
-                $Selectors.Add($Selector) | Out-Null
-            }
-            $DkimAnalysis.MailProvider = $MXRecord.MailProvider
-            if ($MXRecord.MailProvider.PSObject.Properties.Name -contains 'MinimumSelectorPass') {
-                $MinimumSelectorPass = $MXRecord.MailProvider.MinimumSelectorPass
-            }
-            $DkimAnalysis.Selectors = $Selectors
+    # MX lookup, check for defined selectors
+    try {
+        $MXRecord = Read-MXRecord -Domain $Domain
+        foreach ($Selector in $MXRecord.Selectors) {
+            $Selectors.Add($Selector) | Out-Null
         }
-        catch {}
-
-        # Explicitly defined DKIM selectors
-        if (Test-Path 'Config\DkimSelectors') {
-            try {
-                Get-ChildItem 'Config\DkimSelectors' -Filter "$($Domain).json" -ErrorAction Stop | ForEach-Object {
-                    try {
-                        $CustomSelectors = Get-Content $_ | ConvertFrom-Json
-                        foreach ($Selector in $CustomSelectors) {
-                            $Selectors.Add($Selector) | Out-Null
-                        }
-                    } 
-                    catch {}
-                }
-            }
-            catch {}
+        $DkimAnalysis.MailProvider = $MXRecord.MailProvider
+        if ($MXRecord.MailProvider.PSObject.Properties.Name -contains 'MinimumSelectorPass') {
+            $MinimumSelectorPass = $MXRecord.MailProvider.MinimumSelectorPass
         }
+        $DkimAnalysis.Selectors = $Selectors
     }
+    catch {}
+
+    # Get unique selectors
+    $Selectors = $Selectors | Sort-Object -Unique
 
     if (($Selectors | Measure-Object | Select-Object -ExpandProperty Count) -gt 0) {
         foreach ($Selector in $Selectors) {
@@ -1469,7 +1454,7 @@ function Read-WhoisRecord {
         foreach ($RegistrarProp in $RegistrarProps) {
             if ($Results.Contains($RegistrarProp)) {
                 $Results._Registrar = $Results.$RegistrarProp
-                if($Results.$RegistrarProp -eq 'Registrar') {
+                if ($Results.$RegistrarProp -eq 'Registrar') {
                     break  # Means we always favour Registrar if it exists, or keep looking
                 }
             }

diff --git a/DomainAnalyser_All/function.json b/DomainAnalyser_All/function.json
@@ -1,9 +1,9 @@
 {
   "bindings": [
     {
-      "name": "tenant",
+      "name": "DomainObject",
       "direction": "in",
       "type": "activityTrigger"
     }
   ]
-}
+}
diff --git a/DomainAnalyser_All/run.ps1 b/DomainAnalyser_All/run.ps1
@@ -1,22 +1,15 @@
-param($tenant)
+param($DomainObject)
 
 Import-Module '.\DNSHelper.psm1'
 
-$Domain = $Tenant.Domain
+$Domain = $DomainObject.rowKey
 
 Log-request -API 'DomainAnalyser' -tenant $tenant.tenant -message "Starting Processing of $($Tenant.Domain)" -sev Debug
 $Result = [PSCustomObject]@{
-    Tenant               = $tenant.tenant
-    GUID                 = $($Tenant.Domain.Replace('.', ''))
+    Tenant               = $DomainObject.partitionKey
+    GUID                 = $($Domain.Replace('.', ''))
     LastRefresh          = $(Get-Date (Get-Date).ToUniversalTime() -UFormat '+%Y-%m-%dT%H:%M:%S.000Z')
     Domain               = $Domain
-    AuthenticationType   = $Tenant.authenticationType
-    IsAdminManaged       = $Tenant.isAdminManaged
-    IsDefault            = $Tenant.isDefault
-    IsInitial            = $Tenant.isInitial
-    IsRoot               = $Tenant.isRoot
-    IsVerified           = $Tenant.isVerified
-    SupportedServices    = $Tenant.supportedServices
     ExpectedSPFRecord    = ''
     ActualSPFRecord      = ''
     SPFPassAll           = ''
@@ -184,7 +177,14 @@ catch {
 
 # DKIM Check
 try {
-    $DkimRecord = Read-DkimRecord -Domain $Domain
+    $DkimParams = @{
+        Domain = $Domain
+    }
+    if (![string]::IsNullOrEmpty($DomainObject.DkimSelectors)) {
+        $DkimParams.Selectors = $DomainObject.DkimSelectors | ConvertFrom-Json
+    }
+
+    $DkimRecord = Read-DkimRecord @DkimParams
 
     $DkimRecordCount = $DkimRecord.Records | Measure-Object | Select-Object -ExpandProperty Count
     $DkimFailCount = $DkimRecord.ValidationFails | Measure-Object | Select-Object -ExpandProperty Count
@@ -206,7 +206,10 @@ $Result.Score = $ScoreDomain
 $Result.ScorePercentage = [int](($Result.Score / $Result.MaximumScore) * 100)
 $Result.ScoreExplanation = ($ScoreExplanation) -join ', '
 
+
+$DomainObject.DomainAnalyser = ($Result | ConvertTo-Json)
+
 # Final Write to Output
 Log-request -API 'DomainAnalyser' -tenant $tenant.tenant -message "DNS Analyser Finished For $($Result.Domain)" -sev Info
-Write-Output $Result
 
+Write-Output $DomainObject
diff --git a/DomainAnalyser_GetQueue/function.json → ...inAnalyser_GetTenantDomains/function.json b/DomainAnalyser_GetQueue/function.json → ...inAnalyser_GetTenantDomains/function.json
@@ -6,4 +6,4 @@
       "direction": "in"
     }
   ]
-}
+}
diff --git a/DomainAnalyser_GetQueue/run.ps1 → DomainAnalyser_GetTenantDomains/run.ps1 b/DomainAnalyser_GetQueue/run.ps1 → DomainAnalyser_GetTenantDomains/run.ps1
@@ -5,7 +5,7 @@ $Tenants = Get-Tenants
 $object = foreach ($Tenant in $Tenants) {
     # Get Domains to Lookup
     try {
-        $Domains = New-GraphGetRequest -uri 'https://graph.microsoft.com/v1.0/domains' -tenantid $Tenant.defaultDomainName | Where-Object { ($_.id -notlike '*.onmicrosoft.com' -and $_.id -NotLike '*.exclaimer.cloud') }
+        $Domains = New-GraphGetRequest -uri 'https://graph.microsoft.com/v1.0/domains' -tenantid $Tenant.defaultDomainName | Where-Object { ($_.id -notlike '*.onmicrosoft.com' -and $_.id -notlike '*.microsoftonline.com' -and $_.id -NotLike '*.exclaimer.cloud' -and $_.id -NotLike '*.codetwo.online') }
         foreach ($d in $domains) {
             [PSCustomObject]@{
                 Tenant             = $Tenant.defaultDomainName

diff --git a/DomainAnalyser_List/run.ps1 b/DomainAnalyser_List/run.ps1
@@ -6,16 +6,84 @@ param($Request, $TriggerMetadata)
 $APIName = $TriggerMetadata.FunctionName
 Log-Request -user $request.headers.'x-ms-client-principal' -API $APINAME -message 'Accessed this API' -Sev 'Debug'
 
-
 # Write to the Azure Functions log stream.
 Write-Host 'PowerShell HTTP trigger function processed a request.'
 
+$DomainTable = Get-CIPPTable -Table 'Domains'
+
 # Get all the things
-$UnfilteredResults = Get-ChildItem '.\Cache_DomainAnalyser\*.json' | ForEach-Object { Get-Content $_.FullName | Out-String | ConvertFrom-Json }
+
+# Convert file json results to table results
+if (Test-Path .\Cache_DomainAnalyser) {
+    $UnfilteredResults = Get-ChildItem '.\Cache_DomainAnalyser\*.json' | ForEach-Object { Get-Content $_.FullName | Out-String }
+
+    foreach ($Result in $UnfilteredResults) { 
+        $Object = $Result | ConvertFrom-Json
+
+        $ExistingDomain = @{
+            Table        = $DomainTable
+            rowKey       = $Object.Domain
+            partitionKey = $Object.Tenant
+        }
+
+        $Domain = Get-AzTableRow @ExistingDomain
+
+        if (!$Domain) {
+            Write-Host 'Adding domain from cache file'
+            $DomainObject = @{
+                Table        = $DomainTable
+                rowKey       = $Object.Domain
+                partitionKey = $Object.Tenant
+                property     = @{
+                    DomainAnalyser = $Result
+                    TenantDetails  = ''
+                    DkimSelectors  = ''
+                    MailProviders  = ''
+                }
+            }
+            Add-AzTableRow @DomainObject | Out-Null
+        }
+        else {
+            Write-Host 'Updating domain from cache file'
+            $Domain.DomainAnalyser = $Result
+            $Domain | Update-AzTableRow -Table $DomainTable | Out-Null
+        }
+        Remove-Item -Path ".\Cache_DomainAnalyser\$($Object.Domain).DomainAnalysis.json" | Out-Null
+    }
+}
 
 # Need to apply exclusion logic
 $Skiplist = Get-Content 'ExcludedTenants' | ConvertFrom-Csv -Delimiter '|' -Header 'Name', 'User', 'Date'
-$Results = $UnfilteredResults | ForEach-Object { $_.GUID = $_.GUID -replace '[^a-zA-Z-]', ''; $_ } | Where-Object { ($Request.Query.tenantFilter -eq 'AllTenants' -and $_.Tenant -notin $Skiplist.Name) -or ($Request.Query.tenantFilter -ne 'AllTenants' -and $_.Tenant -eq $Request.Query.tenantFilter) }
+
+$DomainList = @{
+    Table        = $DomainTable
+    SelectColumn = @('partitionKey', 'DomainAnalyser')
+}
+
+if ($Request.Query.tenantFilter -ne 'AllTenants') {
+    $DomainList.partitionKey = $Request.Query.tenantFilter
+}
+
+try {
+    # Extract json from table results
+    $Results = foreach ($DomainAnalyserResult in (Get-AzTableRow @DomainList).DomainAnalyser) {
+        try { 
+            if (![string]::IsNullOrEmpty($DomainAnalyserResult)) {
+                $Object = $DomainAnalyserResult | ConvertFrom-Json
+
+                if (($Request.Query.tenantFilter -eq 'AllTenants' -and $Object.Tenant -notin $Skiplist.Name) -or $Request.Query.tenantFilter -ne 'AllTenants') {
+                    $Object.GUID = $Object.GUID -replace '[^a-zA-Z-]', ''
+                    $Object
+                }
+            }
+        }
+        catch {}
+    }
+}
+catch {
+    $Results = @()
+}
+
 
 # Associate values to output bindings by calling 'Push-OutputBinding'.
 Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{

diff --git a/DomainAnalyser_Orchestration/run.ps1 b/DomainAnalyser_Orchestration/run.ps1
@@ -1,21 +1,67 @@
 param($Context)
 
-New-Item "Cache_DomainAnalyser" -ItemType Directory -ErrorAction SilentlyContinue
-New-Item "Cache_DomainAnalyser\CurrentlyRunning.txt" -ItemType File -Force
-$Batch = (Invoke-ActivityFunction -FunctionName 'DomainAnalyser_GetQueue' -Input 'LetsGo')
-$ParallelTasks = foreach ($Item in $Batch) {
-  Invoke-DurableActivity -FunctionName "DomainAnalyser_All" -Input $item -NoWait
-}
+try { 
+  New-Item 'Cache_DomainAnalyser' -ItemType Directory -ErrorAction SilentlyContinue
+  New-Item 'Cache_DomainAnalyser\CurrentlyRunning.txt' -ItemType File -Force
 
-$Outputs = Wait-ActivityFunction -Task $ParallelTasks
-Log-request -API "DomainAnalyser" -tenant $tenant -message "Outputs found count = $($Outputs.count)" -sev Info
+  $DomainTable = Get-CippTable -Table Domains
 
-foreach ($item in $Outputs) {
-  Write-Host $Item | Out-String
-  $Object = $Item | ConvertTo-Json
+  $TenantDomains = Invoke-ActivityFunction -FunctionName 'DomainAnalyser_GetTenantDomains' -Input 'Tenants'
 
-  Set-Content "Cache_DomainAnalyser\$($item.domain).DomainAnalysis.json" -Value $Object -Force
-}
+  # Process tenant domain results
+  foreach ($Tenant in $TenantDomains) {
+    $TenantDetails = $Tenant | ConvertTo-Json
+
+    $ExistingDomain = @{
+      Table        = $DomainTable
+      rowKey       = $Tenant.Domain
+      partitionKey = $Tenant.Tenant
+    }
+    $Domain = Get-AzTableRow @ExistingDomain
+
+    if (!$Domain) {
+      $DomainObject = @{
+        Table        = $DomainTable
+        rowKey       = $Tenant.Domain
+        partitionKey = $Tenant.Tenant
+        property     = @{
+          DomainAnalyser = ''
+          TenantDetails  = $TenantDetails
+          DkimSelectors  = ''
+          MailProviders  = ''
+        }
+      }
+      Add-AzTableRow @DomainObject | Out-Null
+    }
+    else {
+      $Domain.TenantDetails = $TenantDetails
+      $Domain | Update-AzTableRow -Table $DomainTable | Out-Null
+    }
+  }
+
+  # Get list of all domains to process
+  $DomainParam = @{
+    Table = $DomainTable
+  }
+
+  $Batch = Get-AzTableRow @DomainParam
 
-Log-request  -API "DomainAnalyser" -tenant $tenant -message "Domain Analyser has Finished" -sev Info
-Remove-Item "Cache_DomainAnalyser\CurrentlyRunning.txt" -Force
+  $ParallelTasks = foreach ($Item in $Batch) {
+    Invoke-DurableActivity -FunctionName 'DomainAnalyser_All' -Input $item -NoWait
+  }
+
+  $Outputs = Wait-ActivityFunction -Task $ParallelTasks
+  Log-request -API 'DomainAnalyser' -message "Outputs found count = $($Outputs.count)" -sev Info
+
+  foreach ($DomainObject in $Outputs) {
+    [PSCustomObject]$DomainObject | Update-AzTableRow @DomainParam | Out-Null
+  }
+}
+catch {
+  Log-request -API 'DomainAnalyser' -message "Domain Analyser Orchestrator Error $($_.Exception.Message)" -sev info
+  Write-Host $_.Exception | ConvertTo-Json
+}
+finally {
+  Log-request -API 'DomainAnalyser' -message 'Domain Analyser has Finished' -sev Info
+  Remove-Item 'Cache_DomainAnalyser\CurrentlyRunning.txt' -Force
+}
diff --git a/DomainAnalyser_OrchestrationStarter/run.ps1 b/DomainAnalyser_OrchestrationStarter/run.ps1
@@ -1,17 +1,17 @@
 using namespace System.Net
 
 param($Request, $TriggerMetadata)
-$CurrentlyRunning = Get-Item "Cache_DomainAnalyser\CurrentlyRunning.txt" -ErrorAction SilentlyContinue | Where-Object -Property LastWriteTime -GT (Get-Date).AddHours(-24)
+$CurrentlyRunning = Get-Item 'Cache_DomainAnalyser\CurrentlyRunning.txt' -ErrorAction SilentlyContinue | Where-Object -Property LastWriteTime -GT (Get-Date).AddHours(-24)
 if ($CurrentlyRunning) {
-    $Results = [pscustomobject]@{"Results" = "Already running. Please wait for the current instance to finish" }
-    Log-request  -API "DomainAnalyser" -message "Attempted to start domain analysis but an instance was already running." -sev Info
+    $Results = [pscustomobject]@{'Results' = 'Already running. Please wait for the current instance to finish' }
+    Log-request -API 'DomainAnalyser' -message 'Attempted to start domain analysis but an instance was already running.' -sev Info
 }
 else {
     $InstanceId = Start-NewOrchestration -FunctionName 'DomainAnalyser_Orchestration'
     Write-Host "Started orchestration with ID = '$InstanceId'"
     $Orchestrator = New-OrchestrationCheckStatusResponse -Request $Request -InstanceId $InstanceId
-    Log-request  -API "DomainAnalyser" -message "Started retrieving domain information" -sev Info
-    $Results = [pscustomobject]@{"Results" = "Started running analysis" }
+    Log-request -API 'DomainAnalyser' -message 'Started retrieving domain information' -sev Info
+    $Results = [pscustomobject]@{'Results' = 'Started running analysis' }
 }
 Write-Host ($Orchestrator | ConvertTo-Json)
 

diff --git a/ListDomainHealth/run.ps1 b/ListDomainHealth/run.ps1
@@ -5,6 +5,8 @@ param($Request, $TriggerMetadata)
 
 Import-Module .\DNSHelper.psm1
 
+$UserCreds = ([System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($request.headers.'x-ms-client-principal')) | ConvertFrom-Json)
+
 $APIName = $TriggerMetadata.FunctionName
 Log-Request -user $request.headers.'x-ms-client-principal' -API $APINAME -message 'Accessed this API' -Sev 'Debug'
 
@@ -15,6 +17,14 @@ $StatusCode = [HttpStatusCode]::OK
 try {
     if ($Request.Query.Action) {
         if ($Request.Query.Domain -match '^(((?!-))(xn--|_{1,1})?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})$') {
+            $DomainTable = Get-CIPPTable -Table 'Domains'
+            $DomainQuery = @{
+                Table      = $DomainTable
+                ColumnName = 'RowKey'
+                Value      = $Request.Query.Domain
+                Operator   = 'Equal'
+            }
+            $DomainInfo = Get-AzTableRow @DomainQuery
             switch ($Request.Query.Action) {
                 'ReadSpfRecord' {
                     $SpfQuery = @{
@@ -40,6 +50,14 @@ try {
                     }
                     if ($Request.Query.Selector) {
                         $DkimQuery.Selectors = ($Request.Query.Selector).trim() -split '\s*,\s*'
+
+                        if ('admin' -in $UserCreds.userRoles -or 'editor' -in $UserCreds.userRoles) {
+                            $DomainInfo.DkimSelectors = ($DkimQuery.Selectors | ConvertTo-Json)
+                            $DomainInfo | Update-AzTableRow -Table $DomainTable
+                        }
+                    }
+                    elseif (![string]::IsNullOrEmpty($DomainInfo.DkimSelectors)) {
+                        $DkimQuery.Selectors = ($DomainInfo.DkimSelectors | ConvertFrom-Json)
                     }
                     $Body = Read-DkimRecord @DkimQuery
                 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -6,4 +6,4 @@ @@
           "direction": "in"
         }
       ]
-    }
+    }