[main] Update dependencies from dotnet/arcade

Directory.Build.targets

@@ -1,6 +1,10 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project>
-
+  <PropertyGroup>
+    <!-- Temporary workaround for Arcade issue in net9-preview5 -->
+    <_NetFrameworkHostedCompilersVersion Condition="'$(_NetFrameworkHostedCompilersVersion)' == ''">4.11.0-3.24280.3</_NetFrameworkHostedCompilersVersion>
+  </PropertyGroup>
+
   <Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
 
   <!--

eng/Version.Details.xml

@@ -44,14 +44,14 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24306.4">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24352.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>7507f80c8db285bbc9939c1dff522a761cf4edc0</Sha>
+      <Sha>4a7d983f833d6b86365ea1b2b4d6ee72fbdbf944</Sha>
     </Dependency>
     <!-- Intermediate is necessary for source build. -->
-    <Dependency Name="Microsoft.SourceBuild.Intermediate.arcade" Version="9.0.0-beta.24306.4">
+    <Dependency Name="Microsoft.SourceBuild.Intermediate.arcade" Version="9.0.0-beta.24352.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>7507f80c8db285bbc9939c1dff522a761cf4edc0</Sha>
+      <Sha>4a7d983f833d6b86365ea1b2b4d6ee72fbdbf944</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
     <Dependency Name="Microsoft.DiaSymReader.Pdb2Pdb" Version="1.1.0-beta2-19575-01">
@@ -62,9 +62,9 @@
       <Uri>https://github.com/dotnet/symreader-converter</Uri>
       <Sha>c5ba7c88f92e2dde156c324a8c8edc04d9fa4fe0</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.24306.4">
+    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.24352.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>7507f80c8db285bbc9939c1dff522a761cf4edc0</Sha>
+      <Sha>4a7d983f833d6b86365ea1b2b4d6ee72fbdbf944</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/common/SetupNugetSources.ps1

@@ -1,32 +1,31 @@
 # This script adds internal feeds required to build commits that depend on internal package sources. For instance,
 # dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables
 # disabled internal Maestro (darc-int*) feeds.
-# 
-# Optionally, this script also adds a credential entry for each of the internal feeds if supplied. This credential
-# is added via the standard environment variable VSS_NUGET_EXTERNAL_FEED_ENDPOINTS. See
-# https://github.com/microsoft/artifacts-credprovider/tree/v1.1.1?tab=readme-ov-file#environment-variables for more details
+#
+# Optionally, this script also adds a credential entry for each of the internal feeds if supplied.
 #
 # See example call for this script below.
 #
 #  - task: PowerShell@2
-#    displayName: Setup Internal Feeds
+#    displayName: Setup Private Feeds Credentials
 #    condition: eq(variables['Agent.OS'], 'Windows_NT')
 #    inputs:
 #      filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
-#      arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config
-#  - task: NuGetAuthenticate@1
-# 
+#      arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+#    env:
+#      Token: $(dn-bot-dnceng-artifact-feeds-rw)
+#
 # Note that the NuGetAuthenticate task should be called after SetupNugetSources.
 # This ensures that:
 # - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt)
-# - The credential provider is installed
+# - The credential provider is installed.
 #
 # This logic is also abstracted into enable-internal-sources.yml.
 
 [CmdletBinding()]
 param (
     [Parameter(Mandatory = $true)][string]$ConfigFile,
-    [string]$Password
+    $Password
 )
 
 $ErrorActionPreference = "Stop"
@@ -35,23 +34,12 @@ Set-StrictMode -Version 2.0
 
 . $PSScriptRoot\tools.ps1
 
-$feedEndpoints = $null
-
-# If a credential is provided, ensure that we don't overwrite the current set of
-# credentials that may have been provided by a previous call to the credential provider.
-if ($Password -and $null -ne $env:VSS_NUGET_EXTERNAL_FEED_ENDPOINTS) {
-    $feedEndpoints = $env:VSS_NUGET_EXTERNAL_FEED_ENDPOINTS | ConvertFrom-Json
-} elseif ($Password) {
-    $feedEndpoints = @{ endpointCredentials = @() }
-}
-
 # Add source entry to PackageSources
-function AddPackageSource($sources, $SourceName, $SourceEndPoint, $pwd) {
+function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) {
     $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']")
 
-    if ($null -eq $packageSource)
+    if ($packageSource -eq $null)
     {
-        Write-Host "`tAdding package source" $SourceName
         $packageSource = $doc.CreateElement("add")
         $packageSource.SetAttribute("key", $SourceName)
         $packageSource.SetAttribute("value", $SourceEndPoint)
@@ -61,33 +49,63 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $pwd) {
         Write-Host "Package source $SourceName already present."
     }
 
-    if ($pwd) {
-        $feedEndpoints.endpointCredentials = AddCredential -endpointCredentials $feedEndpoints.endpointCredentials -source $SourceEndPoint -pwd $pwd
-    }
+    AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd
 }
 
-# Add a new feed endpoint credential
-function AddCredential([array]$endpointCredentials, $source, $pwd) {
-    $endpointCredentials += @{
-        endpoint = $source;
-        password = $pwd
+# Add a credential node for the specified source
+function AddCredential($creds, $source, $username, $pwd) {
+    # If no cred supplied, don't do anything.
+    if (!$pwd) {
+        return;
     }
-    return $endpointCredentials
+
+    # Looks for credential configuration for the given SourceName. Create it if none is found.
+    $sourceElement = $creds.SelectSingleNode($Source)
+    if ($sourceElement -eq $null)
+    {
+        $sourceElement = $doc.CreateElement($Source)
+        $creds.AppendChild($sourceElement) | Out-Null
+    }
+
+    # Add the <Username> node to the credential if none is found.
+    $usernameElement = $sourceElement.SelectSingleNode("add[@key='Username']")
+    if ($usernameElement -eq $null)
+    {
+        $usernameElement = $doc.CreateElement("add")
+        $usernameElement.SetAttribute("key", "Username")
+        $sourceElement.AppendChild($usernameElement) | Out-Null
+    }
+    $usernameElement.SetAttribute("value", $Username)
+
+    # Add the <ClearTextPassword> to the credential if none is found.
+    # Add it as a clear text because there is no support for encrypted ones in non-windows .Net SDKs.
+    #   -> https://github.com/NuGet/Home/issues/5526
+    $passwordElement = $sourceElement.SelectSingleNode("add[@key='ClearTextPassword']")
+    if ($passwordElement -eq $null)
+    {
+        $passwordElement = $doc.CreateElement("add")
+        $passwordElement.SetAttribute("key", "ClearTextPassword")
+        $sourceElement.AppendChild($passwordElement) | Out-Null
+    }
+
+    $passwordElement.SetAttribute("value", $pwd)
 }
 
-function InsertMaestroInternalFeedCredentials($Sources, $pwd) {
-    $maestroInternalSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
+function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) {
+    $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
 
-    ForEach ($PackageSource in $maestroInternalSources) {
-        Write-Host "`tAdding credential for Maestro's feed:" $PackageSource.Key
-        $feedEndpoints.endpointCredentials = AddCredential -endpointCredentials $feedEndpoints.endpointCredentials -source $PackageSource.value -pwd $pwd
+    Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds."
+
+    ForEach ($PackageSource in $maestroPrivateSources) {
+        Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key
+        AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd
     }
 }
 
-function EnableInternalPackageSources($DisabledPackageSources) {
-    $maestroInternalSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]")
-    ForEach ($DisabledPackageSource in $maestroInternalSources) {
-        Write-Host "`tEnsuring internal source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource"
+function EnablePrivatePackageSources($DisabledPackageSources) {
+    $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]")
+    ForEach ($DisabledPackageSource in $maestroPrivateSources) {
+        Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource"
         # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries
         $DisabledPackageSources.RemoveChild($DisabledPackageSource)
     }
@@ -105,46 +123,49 @@ $doc.Load($filename)
 
 # Get reference to <PackageSources> or create one if none exist already
 $sources = $doc.DocumentElement.SelectSingleNode("packageSources")
-if ($null -eq $sources) {
+if ($sources -eq $null) {
     $sources = $doc.CreateElement("packageSources")
     $doc.DocumentElement.AppendChild($sources) | Out-Null
 }
 
+$creds = $null
+if ($Password) {
+    # Looks for a <PackageSourceCredentials> node. Create it if none is found.
+    $creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials")
+    if ($creds -eq $null) {
+        $creds = $doc.CreateElement("packageSourceCredentials")
+        $doc.DocumentElement.AppendChild($creds) | Out-Null
+    }
+}
+
 # Check for disabledPackageSources; we'll enable any darc-int ones we find there
 $disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources")
-if ($null -ne $disabledSources) {
+if ($disabledSources -ne $null) {
     Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node"
-    EnableInternalPackageSources -DisabledPackageSources $disabledSources
+    EnablePrivatePackageSources -DisabledPackageSources $disabledSources
 }
 
-if ($Password) {
-    InsertMaestroInternalFeedCredentials -Sources $sources -pwd $Password
-}
+$userName = "dn-bot"
+
+# Insert credential nodes for Maestro's private feeds
+InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password
 
 # 3.1 uses a different feed url format so it's handled differently here
 $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
-if ($null -ne $dotnet31Source) {
-    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json" -pwd $Password
-    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json" -pwd $Password
+if ($dotnet31Source -ne $null) {
+    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password
+    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
 }
 
 $dotnetVersions = @('5','6','7','8')
 
 foreach ($dotnetVersion in $dotnetVersions) {
     $feedPrefix = "dotnet" + $dotnetVersion;
     $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']")
-    if ($dotnetSource) {
-        AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedprefix-internal/nuget/v3/index.json" -pwd $Password
-        AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v3/index.json" -pwd $Password
+    if ($dotnetSource -ne $null) {
+        AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password
+        AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
     }
 }
 
 $doc.Save($filename)
-
-# If any credentials were added or altered, update the VSS_NUGET_EXTERNAL_FEED_ENDPOINTS environment variable
-if ($null -ne $feedEndpoints) {
-    # ci is set to true so vso logging commands will be used.
-    $ci = $true
-    Write-PipelineSetVariable -Name 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' -Value $($feedEndpoints | ConvertTo-Json) -IsMultiJobVariable $false
-    Write-PipelineSetVariable -Name 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED' -Value "False" -IsMultiJobVariable $false
-}

eng/common/SetupNugetSources.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This script adds internal feeds required to build commits that depend on intenral package sources. For instance,
+# This script adds internal feeds required to build commits that depend on internal package sources. For instance,
 # dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables
 # disabled internal Maestro (darc-int*) feeds.
 # 

eng/common/core-templates/job/job.yml

@@ -24,12 +24,11 @@ parameters:
   enablePublishTestResults: false
   enablePublishUsingPipelines: false
   enableBuildRetry: false
-  disableComponentGovernance: ''
-  componentGovernanceIgnoreDirectories: ''
   mergeTestResults: false
   testRunTitle: ''
   testResultsFormat: ''
   name: ''
+  componentGovernanceSteps: []
   preSteps: []
   artifactPublishSteps: []
   runAsPublic: false
@@ -170,17 +169,8 @@ jobs:
         uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }}
       continueOnError: true
 
-  - template: /eng/common/core-templates/steps/component-governance.yml
-    parameters:
-      is1ESPipeline: ${{ parameters.is1ESPipeline }}
-      ${{ if eq(parameters.disableComponentGovernance, '') }}:
-        ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
-          disableComponentGovernance: false
-        ${{ else }}:
-          disableComponentGovernance: true
-      ${{ else }}:
-        disableComponentGovernance: ${{ parameters.disableComponentGovernance }}
-      componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }}
+  - ${{ each step in parameters.componentGovernanceSteps }}:
+    - ${{ step }}
 
   - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
     - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
@@ -190,14 +180,6 @@ jobs:
         continueOnError: ${{ parameters.continueOnError }}
         env:
           TeamName: $(_TeamName)
-  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}:
-    - template: /eng/common/core-templates/steps/generate-sbom.yml
-      parameters:
-        is1ESPipeline: ${{ parameters.is1ESPipeline }}
-        PackageVersion: ${{ parameters.packageVersion}}
-        BuildDropPath: ${{ parameters.buildDropPath }}
-        IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }}
-        publishArtifacts: false
 
   # Publish test results
   - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}:

eng/common/core-templates/job/publish-build-assets.yml

@@ -87,13 +87,15 @@ jobs:
 
     - task: NuGetAuthenticate@1
 
-    - task: PowerShell@2
+    - task: AzureCLI@2
       displayName: Publish Build Assets
       inputs:
-        filePath: eng\common\sdk-task.ps1
+        azureSubscription: "Darc: Maestro Production"
+        scriptType: ps
+        scriptLocation: scriptPath
+        scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1
         arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
           /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
-          /p:BuildAssetRegistryToken=$(MaestroAccessToken)
           /p:MaestroApiEndpoint=https://maestro.dot.net
           /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
           /p:OfficialBuildId=$(Build.BuildNumber)
@@ -153,14 +155,16 @@ jobs:
           PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
           is1ESPipeline: ${{ parameters.is1ESPipeline }}
 
-      - task: PowerShell@2
+      - task: AzureCLI@2
         displayName: Publish Using Darc
         inputs:
-          filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
+          azureSubscription: "Darc: Maestro Production"
+          scriptType: ps
+          scriptLocation: scriptPath
+          scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
           arguments: -BuildId $(BARBuildId) 
             -PublishingInfraVersion 3
             -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
-            -MaestroToken '$(MaestroApiAccessToken)'
             -WaitPublishingFinish true
             -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
             -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'

eng/common/core-templates/job/source-index-stage1.yml

@@ -69,23 +69,11 @@ jobs:
 
   - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
     - task: AzureCLI@2
-      displayName: Get stage 1 auth token
+      displayName: Log in to Azure and upload stage1 artifacts to source index
       inputs:
         azureSubscription: 'SourceDotNet Stage1 Publish'
         addSpnToEnvironment: true
         scriptType: 'ps'
         scriptLocation: 'inlineScript'
         inlineScript: |
-          echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId"
-          echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken"
-          echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId"
-
-    - script: |
-        echo "Client ID: $(ARM_CLIENT_ID)"
-        echo "ID Token: $(ARM_ID_TOKEN)"
-        echo "Tenant ID: $(ARM_TENANT_ID)"
-        az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN)
-      displayName: "Login to Azure"
-
-    - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
-      displayName: Upload stage1 artifacts to source index
+          $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1

eng/common/core-templates/post-build/common-variables.yml

@@ -8,8 +8,6 @@ variables:
   # Default Maestro++ API Endpoint and API Version
   - name: MaestroApiEndPoint
     value: "https://maestro.dot.net"
-  - name: MaestroApiAccessToken
-    value: $(MaestroAccessToken)
   - name: MaestroApiVersion
     value: "2020-02-20"