Hydration updates (#963)

anwather · Anthony Watherston · web-flow · commit 11bebe9cb771 · 2025-05-28T09:09:59.000+10:00
Co-authored-by: Anthony Watherston &lt;Anthony.Watherston@microsoft.com&gt;
diff --git a/Docs/guidance-remediation.md b/Docs/guidance-remediation.md
@@ -60,7 +60,7 @@ The objective at this point is to reduce the administration teams' effort to cor
 
 While Infrastructure as Code is an excellent first layer, the second layer in a Defense In Depth model is to enforce this. In Azure Resource Manager, we use Azure Policy Remediation Tasks to accomplish this. Whereas Start-AzPolicyRemediation does allow very targetted deployments, EPAC seeks to take corrective action in broad swaths using the security structure that has been implemented. To this end, the cmdlet New-AzRemediationTasks was created, and can be used in a pipeline to remediate all policyAssignments (that have that capability) in a single pipeline action that should be scheduled with a cron trigger.
 
-Once the *Updating Security Posture* Workstreams above are complete, it is time to move on into the upper tiers of CMM for ARM Governance, congratulations! The environment will be largely self-correcting after this is compelte.
+Once the *Updating Security Posture* Workstreams above are complete, it is time to move on into the upper tiers of CMM for ARM Governance, congratulations! The environment will be largely self-correcting after this is complete.
 
 1. Change `enforcementMode: 'DoNotEnforce'` to `enforcementMode: 'Default'` where applicable
 1. Ensure `DesireStateConfiguration: 'full'` is configured
diff --git a/Docs/integrating-with-alz-library.md b/Docs/integrating-with-alz-library.md
@@ -101,16 +101,16 @@ The next command will generate policy assignments based on the values in this fi
 
 ```ps1
 # Sync the ALZ policies and assign to the "epac-dev" PAC environment.
-Sync-ALZPolicyFromLibrary -DefinitionsRootFolder .\Definitions -Type ALZ -DefinitionsRootFolder .\Definitions -PacEnvironmentSelector "epac-dev"
+Sync-ALZPolicyFromLibrary -DefinitionsRootFolder .\Definitions -Type ALZ -PacEnvironmentSelector "epac-dev"
 
 # Sync the ALZ policies and assign to the "epac-dev" PAC environment. Specify a tagged version of the ALZ library
-Sync-ALZPolicyFromLibrary -DefinitionsRootFolder .\Definitions -Type ALZ -DefinitionsRootFolder .\Definitions -PacEnvironmentSelector "epac-dev" -Tag "platform/alz/2025.02.0"
+Sync-ALZPolicyFromLibrary -DefinitionsRootFolder .\Definitions -Type ALZ -PacEnvironmentSelector "epac-dev" -Tag "platform/alz/2025.02.0"
 
 # Sync the ALZ policies from a cloned/modified library
-Sync-ALZPolicyFromLibrary -DefinitionsRootFolder .\Definitions -Type ALZ -DefinitionsRootFolder .\Definitions -PacEnvironmentSelector "epac-dev" -LibraryPath <<path to library>>
+Sync-ALZPolicyFromLibrary -DefinitionsRootFolder .\Definitions -Type ALZ -PacEnvironmentSelector "epac-dev" -LibraryPath <<path to library>>
 
 # Sync the AMBA policies and assign to the "epac-dev" PAC environment.
-Sync-ALZPolicyFromLibrary -DefinitionsRootFolder .\Definitions -Type AMBA -DefinitionsRootFolder .\Definitions -PacEnvironmentSelector "epac-dev"
+Sync-ALZPolicyFromLibrary -DefinitionsRootFolder .\Definitions -Type AMBA -PacEnvironmentSelector "epac-dev"
 ```
 
 Carefully review the generated policy assigments and ensure all parameter and scope information is correct.
diff --git a/Scripts/CloudAdoptionFramework/New-ALZPolicyDefaultStructure.ps1 b/Scripts/CloudAdoptionFramework/New-ALZPolicyDefaultStructure.ps1
@@ -62,12 +62,24 @@ foreach ($mg in $archetypeDefinitionFile.management_groups) {
 $policyDefaultFile = Get-Content -Path "$LibraryPath\platform\$($Type.ToLower())\alz_policy_default_values.json" | ConvertFrom-Json
 
 foreach ($parameter in $policyDefaultFile.defaults) {
+    # Grab the first policy assignment to grab default value of the parameter
+    $parameterAssignmentName = $parameter.policy_assignments[0].parameter_names[0]
+    $assignment = $parameter.policy_assignments[0]
+
+    $assingmentFileName = ("$($assignment.policy_assignment_name).alz_policy_assignment.json")
+    if ($type -eq "AMBA") {
+        $assingmentFileName = $assingmentFileName -replace ("-", "_")
+    }
+    $file = Get-ChildItem -Recurse -Path ".\temp" -Filter "$assingmentFileName" -File | Select-Object -First 1
+    $jsonContent = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json
+    $tempDefaultParamValue = $jsonContent.properties.parameters.$parameterAssignmentName.value
+    
     $obj = @{
         description            = $parameter.description
         policy_assignment_name = $parameter.policy_assignments.policy_assignment_name
         parameters             = @{
             parameter_name = $parameter.policy_assignments[0].parameter_names[0]
-            value          = ""
+            value          = $tempDefaultParamValue
         }
     }
 
diff --git a/Scripts/Helpers/Add-HelperScripts.ps1 b/Scripts/Helpers/Add-HelperScripts.ps1
@@ -1,5 +1,8 @@
 #Requires -PSEdition Core
 
+# Define ./Scripts location in repo based on it being the parent of the Helpers directiory in which this script is located
+$scriptRoot = Split-Path $PSScriptRoot -Parent
+
 # Load cmdlets
 . "$PSScriptRoot/Add-ErrorMessage.ps1"
 . "$PSScriptRoot/Add-SelectedPacArray.ps1"
@@ -120,3 +123,48 @@
 . "$PSScriptRoot/RestMethods/Set-AzPolicySetDefinitionRestMethod.ps1"
 . "$PSScriptRoot/RestMethods/Set-AzPolicyExemptionRestMethod.ps1"
 . "$PSScriptRoot/RestMethods/Set-AzRoleAssignmentRestMethod.ps1"
+
+# Hydration Kit Content
+
+. "$PSScriptRoot/Build-HydrationAssignmentPlan.ps1"
+. "$PSScriptRoot/Build-HydrationPolicyPlan.ps1"
+. "$PSScriptRoot/Build-HydrationPolicySetPlan.ps1"
+. "$PSScriptRoot/Compare-HydrationMetadata.ps1"
+. "$PSScriptRoot/Copy-HydrationOrderedHashtable.ps1"
+. "$PSScriptRoot/Export-HydrationObjectToJsonFile.ps1"
+. "$PSScriptRoot/Get-HydrationChildManagementGroupNameList.ps1"
+. "$PSScriptRoot/Get-HydrationDefinitionSubfolderByContentId.ps1"
+. "$PSScriptRoot/Get-HydrationEpacRepo.ps1"
+. "$PSScriptRoot/Get-HydrationUserObjectId.ps1"
+. "$PSScriptRoot/Join-HydrationHashtableToPath.ps1"
+. "$PSScriptRoot/New-HydrationAnswerFile.ps1"
+. "$PSScriptRoot/New-HydrationAnswerSet.ps1"
+. "$PSScriptRoot/New-HydrationChangeEntry.ps1"
+. "$PSScriptRoot/New-HydrationContinuePrompt.ps1"
+. "$PSScriptRoot/New-HydrationManagementGroupChildren.ps1"
+. "$PSScriptRoot/New-HydrationMenuResponse.ps1"
+. "$PSScriptRoot/New-HydrationMultipleChoicePrompt.ps1"
+. "$PSScriptRoot/New-HydrationSeparatorBlock.ps1"
+. "$PSScriptRoot/Remove-HydrationChildHierarchy.ps1"
+. "$PSScriptRoot/Test-HydrationAccess.ps1"
+. "$PSScriptRoot/Test-HydrationCaf3Hierarchy.ps1"
+. "$PSScriptRoot/Update-HydrationModuleToSupportedVersion.ps1"
+. "$PSScriptRoot/Update-HydrationStarterKitAssignmentScope.ps1"
+. "$PSScriptRoot/Write-HydrationLogFile.ps1"
+
+. "$scriptRoot/HydrationKit/Build-HydrationDeploymentPlans.ps1"
+. "$scriptRoot/HydrationKit/Copy-HydrationManagementGroupHierarchy.ps1"
+. "$scriptRoot/HydrationKit/Install-HydrationEpac.ps1"
+. "$scriptRoot/HydrationKit/New-FilteredExceptionFile.ps1"
+. "$scriptRoot/HydrationKit/New-HydrationAssignmentPacSelector.ps1"
+. "$scriptRoot/HydrationKit/New-HydrationCaf3Hierarchy.ps1"
+. "$scriptRoot/HydrationKit/New-HydrationDefinitionsFolder.ps1"
+. "$scriptRoot/HydrationKit/New-HydrationGlobalSettingsFile.ps1"
+. "$scriptRoot/HydrationKit/New-HydrationPolicyDocumentationSourceFile.ps1"
+. "$scriptRoot/HydrationKit/Remove-HydrationManagementGroupRecursively.ps1"
+. "$scriptRoot/HydrationKit/Test-HydrationConnection.ps1"
+. "$scriptRoot/HydrationKit/Test-HydrationManagementGroupName.ps1"
+. "$scriptRoot/HydrationKit/Test-HydrationPath.ps1"
+. "$scriptRoot/HydrationKit/Test-HydrationRbacAssignment.ps1"
+. "$scriptRoot/HydrationKit/Update-HydrationAssignmentDestination.ps1"
+. "$scriptRoot/HydrationKit/Update-HydrationDefinitionFolderStructureByAssignment.ps1"
diff --git a/Scripts/Helpers/Build-HydrationAssignmentPlan.ps1 b/Scripts/Helpers/Build-HydrationAssignmentPlan.ps1
@@ -326,6 +326,9 @@ function Build-HydrationAssignmentPlan {
                             $assignmentRecord.Set_Item('metadataChanged', !($metadataMatches))
                             $assignmentRecord.Set_Item('oldMetadata', $deployedPolicyAssignmentProperties.metadata)
                             $assignmentRecord.Set_Item('newMetadata', $metadata)
+                            $metadataComparison = Compare-HydrationMetadata -oldKeys $deployedPolicyAssignmentProperties.metadata -newKeys $metadata
+                            $assignmentRecord.Set_Item('metadataComparison', $metadataComparison)
+
                         }          
                         if (!$definitionVersionMatches) {
                             $assignmentRecord.Set_Item('definitionVersionChanged', !($definitionVersionMatches))
diff --git a/Scripts/Helpers/Build-HydrationPolicyPlan.ps1 b/Scripts/Helpers/Build-HydrationPolicyPlan.ps1
@@ -245,6 +245,8 @@ function Build-HydrationPolicyPlan {
                         $fileRecord.Set_Item('metadataChanged', !($metadataMatches))
                         $fileRecord.Set_Item('oldMetadata', $deployedDefinitionProperties.metadata)
                         $fileRecord.Set_Item('newMetadata', $metadata)
+                        $metadataComparison = Compare-HydrationMetadata -oldKeys $deployedDefinitionProperties.metadata -newKeys $metadata
+                        $fileRecord.Set_Item('metadataComparison', $metadataComparison)
                     }
                     if (!$parametersMatch -and !$incompatible) {
                         $fileRecord.Set_Item('parametersChanged', !($parametersMatch))
diff --git a/Scripts/Helpers/Build-HydrationPolicySetPlan.ps1 b/Scripts/Helpers/Build-HydrationPolicySetPlan.ps1
@@ -227,7 +227,7 @@ function Build-HydrationPolicySetPlan {
             $parametersMatch, $incompatible = Confirm-ParametersDefinitionMatch `
                 -ExistingParametersObj $deployedDefinition.parameters `
                 -DefinedParametersObj $parameters
-            Remove-Variable policyDefinitionsMatch -ErrorAction SilentlyContinue
+            Remove-Variable policyDefinitionsMatch -ErrorAction SilentlyContinue # TODO: THis is where it breaks
             $policyDefinitionsMatch = Confirm-PolicyDefinitionsInPolicySetMatch `
                 $deployedDefinition.policyDefinitions `
                 $policyDefinitionsFinal
@@ -238,7 +238,9 @@ function Build-HydrationPolicySetPlan {
             $deletedPolicyDefinitionGroups = !$policyDefinitionGroupsMatch -and ($null -eq $policyDefinitionGroupsFinal -or $policyDefinitionGroupsFinal.Length -eq 0)            
             # Update Policy Set in Azure if necessary
             $containsReplacedPolicy = $false
-            $replacedPolicyList = @()
+            if($ExtendedReporting){
+                $replacedPolicyList = @()
+            }
             foreach ($policyDefinitionEntry in $policyDefinitionsFinal) {
                 $policyId = $policyDefinitionEntry.policyDefinitionId
                 if ($ReplaceDefinitions.ContainsKey($policyId)) {
@@ -249,6 +251,7 @@ function Build-HydrationPolicySetPlan {
                     else{
                         # Capture full list of replaced policies for ExtendedReporting
                         $replacedPolicyList += $policyId
+                        break
                     }
                 }
             }
@@ -345,6 +348,14 @@ function Build-HydrationPolicySetPlan {
                         $fileRecord.Set_Item('metadataChanged', !($metadataMatches))
                         $fileRecord.Set_Item('oldMetadata', $deployedDefinition.metadata)
                         $fileRecord.Set_Item('newMetadata', $metadata)
+                        $metadataComparison = Compare-HydrationMetadata -oldKeys $deployedDefinition.metadata -newKeys $metadata
+                        try{
+                            $fileRecord.Set_Item('metadataComparison', $metadataComparison)
+
+                        }
+                        catch{
+                            Write-Error "Failed to set metadataComparison for $($fileRecord.id) with $($metadataComparison)"
+                        }
                     }
                     if (!$parametersMatch -and !$incompatible) { # I don't think this is really useful, we don't test on it anywhere, we test on incompatible... which appears to be the same intended outcome.
                         $fileRecord.Set_Item('parametersChanged', !($parametersMatch))
diff --git a/Scripts/Helpers/Compare-HydrationMetadata.ps1 b/Scripts/Helpers/Compare-HydrationMetadata.ps1
@@ -0,0 +1,43 @@
+function Compare-HydrationMetadata {
+
+    [CmdletBinding()]
+    param (
+        [Parameter(Mandatory = $false)]
+        $oldKeys = @{},
+        [Parameter(Mandatory = $false)]
+        $newKeys = @{}
+    )
+
+    $metadataComparison = [ordered]@{
+        ValueDifferences = @{}
+        OnlyInOld        = @()
+        OnlyInNew        = @()
+    }
+
+    # Compare values for matching keys
+    $valueDifferences = @()
+    foreach ($key in ($oldKeys.keys + $newKeys.keys | Select-Object -Unique)) {
+        $oldValue = $oldKeys[$key]
+        $newValue = $newKeys[$key]
+        if ($oldValue -ne $newValue) {
+        $valueDifferences += `
+            [PSCustomObject]@{
+                Key      = $key
+                OldValue = $oldValue
+                NewValue = $newValue
+            }
+        }
+    }
+
+    # Find keys only in old or only in new
+    $onlyInOld = $oldKeys | Where-Object { $_ -notin $newKeys }
+    $onlyInNew = $newKeys | Where-Object { $_ -notin $oldKeys }
+
+    $metadataComparison = [ordered]@{
+        ValueDifferences = $valueDifferences
+        OnlyInOld        = $onlyInOld
+        OnlyInNew        = $onlyInNew
+    }
+
+    return $metadataComparison
+}
diff --git a/Scripts/Helpers/Confirm-PolicyDefinitionsInPolicySetMatch.ps1 b/Scripts/Helpers/Confirm-PolicyDefinitionsInPolicySetMatch.ps1
@@ -41,7 +41,33 @@ function Confirm-PolicyDefinitionsInPolicySetMatch {
             }
 
             # Validate the Azure definitionVersion with the local definitionVersion, if the local definitionVersion doesn't exist and the Azure definitionVersion is not equal to latest policy version then return false
-            $definitionVersionMatches = Compare-SemanticVersion -Version1 $($item1.definitionVersion ?? $Definitions[$item1.policyDefinitionId].properties.version ?? '1.*.*') -Version2 $($item2.definitionVersion ?? $Definitions[$item1.policyDefinitionId].properties.version ?? '1.*.*')
+            # This addresses an error that occurs when there is a null value in the definitionVersion field that cropped up when we removed the variable prior to processing to fix a bug spotted in Build-HydrationDeploymentPlans where the values were retained, and adversely affecting the update information.
+            try {
+                if ($null -eq $item1.definitionVersion -and $null -eq $item2.definitionVersion) {
+                    # Compare-SemanticVersion -Version1 0 -Version2 0 is always 0, so we forego the calculation and set it
+                    $definitionVersionMatches = 0
+                }
+                elseif ($null -eq $item1.definitionVersion) {
+                    # Compare-SemanticVersion -Version1 0 -Version2 (anything not 0) is always -1, so we forego the calculation and set it
+                    # $definitionVersionMatches = Compare-SemanticVersion -Version1 0 -Version2 $item2.definitionVersion
+                    $definitionVersionMatches = -1
+                }
+                elseif ($null -eq $item2.definitionVersion) {
+                    # Compare-SemanticVersion -Version1 (anything not 0) -Version2 0 is always 1, so we forego the calculation and set it
+                    # $definitionVersionMatches = Compare-SemanticVersion -Version1 $item1.definitionVersion -Version2 0
+                    $definitionVersionMatches = 1
+                }               
+                else {
+                    # If neither of the definitionVersion values are null, then the compare can proceed without error
+                    $definitionVersionMatches = Compare-SemanticVersion -Version1 $($item1.definitionVersion ?? $Definitions[$item1.policyDefinitionId].properties.version ?? '1.*.*') -Version2 $($item2.definitionVersion ?? $Definitions[$item1.policyDefinitionId].properties.version ?? '1.*.*')
+                }
+            }
+            catch {
+                Write-Information "Comparison has generated an error."
+                Write-Information "Item1: $($item1.policyDefinitionId) $($item1.policySetDefinitionId) $($item1.policyDefinitionName) $($item1.policySetDefinitionName)"
+                Write-Information "Item2: $($item2.policyDefinitionId) $($item2.policySetDefinitionId) $($item2.policyDefinitionName) $($item2.policySetDefinitionName)"
+                continue
+            }
             if ($definitionVersionMatches -ne 0) {
                 Write-Verbose "Definition Id: $($item1.policyDefinitionId)"
                 Write-Verbose "DefinitionVersion does not match: Azure: $($item1.definitionVersion), Local: $($item2.definitionVersion)"
diff --git a/Scripts/HydrationKit/Build-HydrationDeploymentPlans.ps1 b/Scripts/HydrationKit/Build-HydrationDeploymentPlans.ps1
@@ -202,6 +202,7 @@ function Build-HydrationDeploymentPlans {
             oldOwner                          = ""
             newOwner                          = ""
             metadataChanged                   = $false
+            metadataComparison                = @{}
             oldMetadata                       = @{}
             newMetadata                       = @{}
             definitionVersionChanged          = $false

Original file line number	Diff line number	Diff line change
`@@ -245,6 +245,8 @@ function Build-HydrationPolicyPlan {`
`245`	`245`	`$fileRecord.Set_Item('metadataChanged', !($metadataMatches))`
`246`	`246`	`$fileRecord.Set_Item('oldMetadata', $deployedDefinitionProperties.metadata)`
`247`	`247`	`$fileRecord.Set_Item('newMetadata', $metadata)`
	`248`	`+ $metadataComparison = Compare-HydrationMetadata -oldKeys $deployedDefinitionProperties.metadata -newKeys $metadata`
	`249`	`+ $fileRecord.Set_Item('metadataComparison', $metadataComparison)`
`248`	`250`	`}`
`249`	`251`	`if (!$parametersMatch -and !$incompatible) {`
`250`	`252`	`$fileRecord.Set_Item('parametersChanged', !($parametersMatch))`