Sync eng/common directory with azure-sdk-tools for PR 10579 (Azure#45269)

* Support writing .env files from Test Resources

If a language repo opts into it *and* if a `test-resources.bicep` file exists and lints clean of writing secrets *and* if the `.env` file is gitignore'd, write a `.env` file next to `test-resources.bicep`.

* Resolve PR feedback

* Pass -Force for . hidden files on non-Windows

---------

Co-authored-by: Heath Stewart <heaths@microsoft.com>

Author: azure-sdk

eng/common/TestResources/New-TestResources.ps1

@@ -122,6 +122,7 @@ param (
     $NewTestResourcesRemainingArguments
 )
 
+. (Join-Path $PSScriptRoot .. scripts common.ps1)
 . (Join-Path $PSScriptRoot .. scripts Helpers Resource-Helpers.ps1)
 . $PSScriptRoot/TestResources-Helpers.ps1
 . $PSScriptRoot/SubConfig-Helpers.ps1
@@ -203,11 +204,14 @@ try {
     $PSBoundParameters['BaseName'] = $BaseName
 
     # Try detecting repos that support OutFile and defaulting to it
-    if (!$CI -and !$PSBoundParameters.ContainsKey('OutFile') -and $IsWindows) {
+    if (!$CI -and !$PSBoundParameters.ContainsKey('OutFile')) {
         # TODO: find a better way to detect the language
-        if (Test-Path "$repositoryRoot/eng/service.proj") {
+        if ($IsWindows -and $Language -eq 'dotnet') {
             $OutFile = $true
-            Log "Detected .NET repository. Defaulting OutFile to true. Test environment settings would be stored into the file so you don't need to set environment variables manually."
+            Log "Detected .NET repository. Defaulting OutFile to true. Test environment settings will be stored into a file so you don't need to set environment variables manually."
+        } elseif ($SupportsTestResourcesDotenv) {
+            $OutFile = $true
+            Log "Repository supports reading .env files. Defaulting OutFile to true. Test environment settings may be stored in a .env file so they are read by tests automatically."
         }
     }
 
@@ -342,10 +346,10 @@ try {
         if ($context.Account.Type -eq 'User') {
             # Support corp tenant and TME tenant user id lookups
             $user = Get-AzADUser -Mail $context.Account.Id
-            if ($user -eq $null -or !$user.Id) {
+            if ($null -eq $user -or !$user.Id) {
                 $user = Get-AzADUser -UserPrincipalName $context.Account.Id
             }
-            if ($user -eq $null -or !$user.Id) {
+            if ($null -eq $user -or !$user.Id) {
                 throw "Failed to find entra object ID for the current user"
             }
             $ProvisionerApplicationOid = $user.Id
@@ -419,10 +423,10 @@ try {
 
         # Support corp tenant and TME tenant user id lookups
         $userAccount = (Get-AzADUser -Mail (Get-AzContext).Account.Id)
-        if ($userAccount -eq $null -or !$userAccount.Id) {
+        if ($null -eq $userAccount -or !$userAccount.Id) {
             $userAccount = (Get-AzADUser -UserPrincipalName (Get-AzContext).Account)
         }
-        if ($userAccount -eq $null -or !$userAccount.Id) {
+        if ($null -eq $userAccount -or !$userAccount.Id) {
             throw "Failed to find entra object ID for the current user"
         }
         $TestApplicationOid = $userAccount.Id
@@ -860,14 +864,19 @@ Force creation of resources instead of being prompted.
 
 .PARAMETER OutFile
 Save test environment settings into a .env file next to test resources template.
-The contents of the file are protected via the .NET Data Protection API (DPAPI).
-This is supported only on Windows. The environment file is scoped to the current
-service directory.
 
+On Windows in the Azure/azure-sdk-for-net repository,
+the contents of the file are protected via the .NET Data Protection API (DPAPI).
+The environment file is scoped to the current service directory.
 The environment file will be named for the test resources template that it was
 generated for. For ARM templates, it will be test-resources.json.env. For
 Bicep templates, test-resources.bicep.env.
 
+If `$SupportsTestResourcesDotenv=$true` in language repos' `LanguageSettings.ps1`,
+and if `.env` files are gitignore'd, and if a service directory's `test-resources.bicep`
+file does not expose secrets based on `bicep lint`, a `.env` file is written next to
+`test-resources.bicep` that can be loaded by a test harness to be used for recording tests.
+
 .PARAMETER SuppressVsoCommands
 By default, the -CI parameter will print out secrets to logs with Azure Pipelines log
 commands that cause them to be redacted. For CI environments that don't support this (like

eng/common/TestResources/New-TestResources.ps1.md

@@ -588,17 +588,19 @@ Accept wildcard characters: False
 
 ### -OutFile
 Save test environment settings into a .env file next to test resources template.
-The contents of the file are protected via the .NET Data Protection API (DPAPI).
-This is supported only on Windows.
-The environment file is scoped to the current
-service directory.
 
+On Windows in the Azure/azure-sdk-for-net repository,
+the contents of the file are protected via the .NET Data Protection API (DPAPI).
+The environment file is scoped to the current service directory.
 The environment file will be named for the test resources template that it was
-generated for.
-For ARM templates, it will be test-resources.json.env.
-For
+generated for. For ARM templates, it will be test-resources.json.env. For
 Bicep templates, test-resources.bicep.env.
 
+If `$SupportsTestResourcesDotenv=$true` in language repos' `LanguageSettings.ps1`,
+and if `.env` files are gitignore'd, and if a service directory's `test-resources.bicep`
+file does not expose secrets based on `bicep lint`, a `.env` file is written next to
+`test-resources.bicep` that can be loaded by a test harness to be used for recording tests.
+
 ```yaml
 Type: SwitchParameter
 Parameter Sets: (All)

eng/common/TestResources/README.md

@@ -1,7 +1,7 @@
 # Live Test Resource Management
 
 Running and recording live tests often requires first creating some resources
-in Azure. Service directories that include a `test-resources.json` or `test-resources.bicep` 
+in Azure. Service directories that include a `test-resources.json` or `test-resources.bicep`
 file require running [New-TestResources.ps1][] to create these resources and output
 environment variables you must set.
 
@@ -19,8 +19,8 @@ scenarios as well as on hosted agents for continuous integration testing.
 ## On the Desktop
 
 To set up your Azure account to run live tests, you'll need to log into Azure,
-and create the resources defined in your `test-resources.json` or `test-resources.bicep` 
-template as shown in the following example using Azure Key Vault. The script will create 
+and create the resources defined in your `test-resources.json` or `test-resources.bicep`
+template as shown in the following example using Azure Key Vault. The script will create
 a service principal automatically, or you may create a service principal that can be reused
 subsequently.
 
@@ -34,12 +34,16 @@ Connect-AzAccount -Subscription 'YOUR SUBSCRIPTION ID'
 eng\common\TestResources\New-TestResources.ps1 keyvault
 ```
 
-The `OutFile` switch will be set by default if you are running this for a .NET project on Windows. 
-This will save test environment settings into a `test-resources.json.env` file next to `test-resources.json` 
+The `OutFile` switch will be set by default if you are running this for a .NET project on Windows.
+This will save test environment settings into a `test-resources.json.env` file next to `test-resources.json`
 or a `test-resources.bicep.env` file next to `test-resources.bicep`. The file is protected via DPAPI.
 The environment file would be scoped to the current repository directory and avoids the need to
 set environment variables or restart your IDE to recognize them.
 
+It will also be set by default for other repositories and on other platforms if your `assets.json`
+file contains `"Dotenv": true`. It must be in your `.gitignore` file;
+otherwise, an error is returned and no file is generated.
+
 Along with some log messages, this will output environment variables based on
 your current shell like in the following example:
 

eng/common/TestResources/Remove-TestResources.ps1

@@ -235,6 +235,7 @@ if ($ServiceDirectory) {
 
     # Make sure environment files from New-TestResources -OutFile are removed.
     Get-ChildItem -Path $root -Filter "$ResourceType-resources.json.env" -Recurse | Remove-Item -Force:$Force
+    Get-ChildItem -Path $root -Filter ".env" -Recurse -Force | Remove-Item -Force
 }
 
 $verifyDeleteScript = {

eng/common/TestResources/TestResources-Helpers.ps1

@@ -149,6 +149,33 @@ function BuildBicepFile([System.IO.FileSystemInfo] $file) {
     return $templateFilePath
 }
 
+function LintBicepFile([string] $path) {
+    if (!(Get-Command bicep -ErrorAction Ignore)) {
+      Write-Error "A bicep file was found at '$path' but the Azure Bicep CLI is not installed. See https://aka.ms/bicep-install"
+      throw
+    }
+
+    # Work around lack of config file override: https://github.com/Azure/bicep/issues/5013
+    $output = bicep lint "$path" 2>&1
+    if ($LASTEXITCODE) {
+        Write-Error "Failed linting bicep file '$path'"
+        throw
+    }
+
+    $clean = $true
+    foreach ($line in $output) {
+        $line = $line.ToString()
+
+        # See https://learn.microsoft.com/azure/azure-resource-manager/bicep/bicep-config-linter for lints.
+        if ($line.Contains('outputs-should-not-contain-secrets')) {
+            $clean = $false
+        }
+        Write-Warning $line
+    }
+
+    $clean
+}
+
 function BuildDeploymentOutputs([string]$serviceName, [object]$azContext, [object]$deployment, [hashtable]$environmentVariables) {
     $serviceDirectoryPrefix = BuildServiceDirectoryPrefix $serviceName
     # Add default values
@@ -203,19 +230,40 @@ function SetDeploymentOutputs(
     $deploymentOutputs = BuildDeploymentOutputs $serviceName $azContext $deployment $deploymentEnvironmentVariables
 
     if ($OutFile) {
-        if (!$IsWindows) {
-            Write-Host 'File option is supported only on Windows'
-        }
+        if ($IsWindows -and $Language -eq 'dotnet') {
+            $outputFile = "$($templateFile.originalFilePath).env"
+
+            $environmentText = $deploymentOutputs | ConvertTo-Json;
+            $bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText)
+            $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser)
 
-        $outputFile = "$($templateFile.originalFilePath).env"
+            Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force
 
-        $environmentText = $deploymentOutputs | ConvertTo-Json;
-        $bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText)
-        $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser)
+            Write-Host "Test environment settings`n$environmentText`nstored into encrypted $outputFile"
+        }
+        elseif ($templateFile.originalFilePath -and $templateFile.originalFilePath.EndsWith(".bicep")) {
+            $bicepTemplateFile = $templateFile.originalFilePath
 
-        Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force
+            # Make sure the file would not write secrets to .env file.
+            if (!(LintBicepFile $bicepTemplateFile)) {
+                Write-Error "$bicepTemplateFile may write secrets. No file written."
+            }
+            $outputFile = $bicepTemplateFile | Split-Path | Join-Path -ChildPath '.env'
 
-        Write-Host "Test environment settings`n $environmentText`nstored into encrypted $outputFile"
+            # Make sure the file would be ignored.
+            git check-ignore -- "$outputFile" > $null
+            if ($?) {
+                $environmentText = foreach ($kv in $deploymentOutputs.GetEnumerator()) {
+                    "$($kv.Key)=`"$($kv.Value)`""
+                }
+
+                Set-Content $outputFile -Value $environmentText -Force
+                Write-Host "Test environment settings`n$environmentText`nstored in $outputFile"
+            }
+            else {
+                Write-Error "$outputFile is not ignored by .gitignore. No file written."
+            }
+        }
     }
     else {
         if (!$CI) {

eng/common/scripts/common.ps1

@@ -24,6 +24,9 @@ $PackageRepository = "Unknown"
 $packagePattern = "Unknown"
 $MetadataUri = "Unknown"
 
+# Whether the language repo supports automatically loading .env file generated from TestResources scripts.
+$SupportsTestResourcesDotenv = $false
+
 # Import common language settings
 $EngScriptsLanguageSettings = Join-path $EngScriptsDir "Language-Settings.ps1"
 if (Test-Path $EngScriptsLanguageSettings) {