Sync eng/common directory with azure-sdk-tools for PR 10038 (#6494)

* add a category to parse triggerPaths when there is no artifact present (this is the go use-case)
* add integration tests for Save-Package-Properties scenarios
* bugfixes around various situations in new PackageProps class

---------

Co-authored-by: Scott Beddall <[email protected]>

Author: azure-sdk

eng/common/pipelines/templates/stages/archetype-sdk-tool-pwsh.yml

@@ -10,6 +10,9 @@ parameters:
   - name: TargetTags
     type: string
     default: ''
+  - name: PreTestSteps
+    type: object
+    default: []
 
 variables:
   - template: /eng/pipelines/templates/variables/globals.yml
@@ -36,6 +39,8 @@ stages:
           vmImage: $(Image)
 
         steps:
+          - ${{ parameters.PreTestSteps }}
+
           - template: /eng/common/pipelines/templates/steps/run-pester-tests.yml
             parameters:
               TargetDirectory: ${{ parameters.TargetDirectory }}

eng/common/scripts/Package-Properties.ps1

@@ -77,11 +77,13 @@ class PackageProps {
         $this.ArtifactName = $artifactName
         $this.Initialize($name, $version, $directoryPath, $serviceDirectory)
     }
-    hidden [PSCustomObject]ParseYmlForArtifact([string]$ymlPath) {
+
+    hidden [PSCustomObject]ParseYmlForArtifact([string]$ymlPath, [bool]$soleCIYml = $false) {
         $content = LoadFrom-Yaml $ymlPath
         if ($content) {
             $artifacts = GetValueSafelyFrom-Yaml $content @("extends", "parameters", "Artifacts")
-            $artifactForCurrentPackage = $null
+            $artifactForCurrentPackage = @{}
+
             if ($artifacts) {
                 # If there's an artifactName match that to the name field from the yml
                 if ($this.ArtifactName) {
@@ -98,8 +100,9 @@ class PackageProps {
                 }
             }
 
-            # if we found an artifact for the current package, we should count this ci file as the source of the matrix for this package
-            if ($artifactForCurrentPackage) {
+            # if we found an artifact for the current package OR this is the sole ci.yml for the given service directory,
+            # we should count this ci file as the source of the matrix for this package
+            if ($artifactForCurrentPackage -or $soleCIYml) {
                 $result = [PSCustomObject]@{
                     ArtifactConfig = [HashTable]$artifactForCurrentPackage
                     ParsedYml = $content
@@ -116,11 +119,12 @@ class PackageProps {
         $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot ".." ".." "..")
 
         $ciFolderPath = Join-Path -Path $RepoRoot -ChildPath (Join-Path "sdk" $this.ServiceDirectory)
-        $ciFiles = Get-ChildItem -Path $ciFolderPath -Filter "ci*.yml" -File
+        $ciFiles = @(Get-ChildItem -Path $ciFolderPath -Filter "ci*.yml" -File)
         $ciArtifactResult = $null
+        $soleCIYml = ($ciFiles.Count -eq 1)
 
         foreach ($ciFile in $ciFiles) {
-            $ciArtifactResult = $this.ParseYmlForArtifact($ciFile.FullName)
+            $ciArtifactResult = $this.ParseYmlForArtifact($ciFile.FullName, $soleCIYml)
             if ($ciArtifactResult) {
                 break
             }
@@ -137,7 +141,7 @@ class PackageProps {
         if (-not $this.ArtifactDetails) {
             $ciArtifactResult = $this.GetCIYmlForArtifact()
 
-            if ($ciArtifactResult) {
+            if ($ciArtifactResult -and $null -ne $ciArtifactResult.ArtifactConfig) {
                 $this.ArtifactDetails = [Hashtable]$ciArtifactResult.ArtifactConfig
 
                 $repoRoot = Resolve-Path (Join-Path $PSScriptRoot ".." ".." "..")
@@ -147,27 +151,32 @@ class PackageProps {
                 if (-not $this.ArtifactDetails["triggeringPaths"]) {
                     $this.ArtifactDetails["triggeringPaths"] = @()
                 }
-                else {
-                    $adjustedPaths = @()
-
-                    # we need to convert relative references to absolute references within the repo
-                    # this will make it extremely easy to compare triggering paths to files in the deleted+changed file list.
-                    for ($i = 0; $i -lt $this.ArtifactDetails["triggeringPaths"].Count; $i++) {
-                        $currentPath = $this.ArtifactDetails["triggeringPaths"][$i]
-                        $newPath = Join-Path $repoRoot $currentPath
-                        if (!$currentPath.StartsWith("/")) {
-                            $newPath = Join-Path $repoRoot $relRoot $currentPath
-                        }
-                        # it is a possibility that users may have a triggerPath dependency on a file that no longer exists.
-                        # before we resolve it to get rid of possible relative references, we should check if the file exists
-                        # if it doesn't, we should just leave it as is. Otherwise we would _crash_ here when a user accidentally
-                        # left a triggeringPath on a file that had been deleted
-                        if (Test-Path $newPath) {
-                            $adjustedPaths += (Resolve-Path -Path $newPath -Relative -RelativeBasePath $repoRoot).TrimStart(".").Replace("`\", "/")
-                        }
+
+                # if we know this is the matrix for our file, we should now see if there is a custom matrix config for the package
+                $serviceTriggeringPaths = GetValueSafelyFrom-Yaml $ciArtifactResult.ParsedYml @("extends", "parameters", "TriggeringPaths")
+                if ($serviceTriggeringPaths){
+                    $this.ArtifactDetails["triggeringPaths"] += $serviceTriggeringPaths
+                }
+
+                $adjustedPaths = @()
+
+                # we need to convert relative references to absolute references within the repo
+                # this will make it extremely easy to compare triggering paths to files in the deleted+changed file list.
+                for ($i = 0; $i -lt $this.ArtifactDetails["triggeringPaths"].Count; $i++) {
+                    $currentPath = $this.ArtifactDetails["triggeringPaths"][$i]
+                    $newPath = Join-Path $repoRoot $currentPath
+                    if (!$currentPath.StartsWith("/")) {
+                        $newPath = Join-Path $repoRoot $relRoot $currentPath
+                    }
+                    # it is a possibility that users may have a triggerPath dependency on a file that no longer exists.
+                    # before we resolve it to get rid of possible relative references, we should check if the file exists
+                    # if it doesn't, we should just leave it as is. Otherwise we would _crash_ here when a user accidentally
+                    # left a triggeringPath on a file that had been deleted
+                    if (Test-Path $newPath) {
+                        $adjustedPaths += (Resolve-Path -Path $newPath -Relative -RelativeBasePath $repoRoot).TrimStart(".").Replace("`\", "/")
                     }
-                    $this.ArtifactDetails["triggeringPaths"] = $adjustedPaths
                 }
+                $this.ArtifactDetails["triggeringPaths"] = $adjustedPaths
                 $this.ArtifactDetails["triggeringPaths"] += $ciYamlPath
 
                 $this.CIParameters["CIMatrixConfigs"] = @()
@@ -215,6 +224,22 @@ function Get-PkgProperties {
     return $null
 }
 
+function Get-PackagesFromPackageInfo([string]$PackageInfoFolder, [bool]$IncludeIndirect, [ScriptBlock]$CustomCompareFunction = $null) {
+    $packages = Get-ChildItem -R -Path $PackageInfoFolder -Filter "*.json" | ForEach-Object {
+        Get-Content $_.FullName | ConvertFrom-Json
+    }
+
+    if (-not $includeIndirect) {
+        $packages = $packages | Where-Object { $_.IncludedForValidation -eq $false }
+    }
+
+    if ($CustomCompareFunction) {
+        $packages = $packages | Where-Object { &$CustomCompareFunction $_ }
+    }
+
+    return $packages
+}
+
 
 function Get-TriggerPaths([PSCustomObject]$AllPackageProperties) {
     $existingTriggeringPaths = @()
@@ -447,7 +472,8 @@ function Get-PrPkgProperties([string]$InputDiffJson) {
     # finally, if we have gotten all the way here and we still don't have any packages, we should include the template service
     # packages. We should never return NO validation.
     if ($packagesWithChanges.Count -eq 0) {
-        $packagesWithChanges += ($allPackageProperties | Where-Object { $_.ServiceDirectory -eq "template" })
+        # most of our languages use `template` as the service directory for the template service, but `go` uses `template/aztemplate`.
+        $packagesWithChanges += ($allPackageProperties | Where-Object { $_.ServiceDirectory -eq "template"-or $_.ServiceDirectory -eq "template/aztemplate" })
         foreach ($package in $packagesWithChanges) {
             $package.IncludedForValidation = $true
         }