[rel/17.10] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -44,14 +44,14 @@
  </Dependency>
  </ProductDependencies>
  <ToolsetDependencies>
- <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24270.3">
+ <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24327.1">
  <Uri>https://github.com/dotnet/arcade</Uri>
- <Sha>1cf3eaa1f6ada43ab988145a3f3efddb1ffa3b10</Sha>
+ <Sha>ede13bd35571c0c8b0c01edcb057031904c5c955</Sha>
  </Dependency>
  <!-- Intermediate is necessary for source build. -->
- <Dependency Name="Microsoft.SourceBuild.Intermediate.arcade" Version="9.0.0-beta.24270.3">
+ <Dependency Name="Microsoft.SourceBuild.Intermediate.arcade" Version="9.0.0-beta.24327.1">
  <Uri>https://github.com/dotnet/arcade</Uri>
- <Sha>1cf3eaa1f6ada43ab988145a3f3efddb1ffa3b10</Sha>
+ <Sha>ede13bd35571c0c8b0c01edcb057031904c5c955</Sha>
  <SourceBuild RepoName="arcade" ManagedOnly="true" />
  </Dependency>
  <Dependency Name="Microsoft.DiaSymReader.Pdb2Pdb" Version="1.1.0-beta2-19575-01">
@@ -62,9 +62,9 @@
  <Uri>https://github.com/dotnet/symreader-converter</Uri>
  <Sha>c5ba7c88f92e2dde156c324a8c8edc04d9fa4fe0</Sha>
  </Dependency>
- <Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.24270.3">
+ <Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.24327.1">
  <Uri>https://github.com/dotnet/arcade</Uri>
- <Sha>1cf3eaa1f6ada43ab988145a3f3efddb1ffa3b10</Sha>
+ <Sha>ede13bd35571c0c8b0c01edcb057031904c5c955</Sha>
  </Dependency>
  </ToolsetDependencies>
 </Dependencies>

eng/common/SetupNugetSources.ps1

@@ -1,17 +1,10 @@
-# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
-# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
+# This script adds internal feeds required to build commits that depend on internal package sources. For instance,
+# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables
+# disabled internal Maestro (darc-int*) feeds.
 #
-# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
-# under <packageSourceCredentials> for each Maestro managed private feed. Two additional credential 
-# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
+# Optionally, this script also adds a credential entry for each of the internal feeds if supplied.
 #
-# This script needs to be called in every job that will restore packages and which the base repo has
-# private AzDO feeds in the NuGet.config.
-#
-# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
-# from the AzureDevOps-Artifact-Feeds-Pats variable group.
-#
-# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing
+# See example call for this script below.
 #
 # - task: PowerShell@2
 # displayName: Setup Private Feeds Credentials
@@ -21,11 +14,18 @@
 # arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
 # env:
 # Token: $(dn-bot-dnceng-artifact-feeds-rw)
+#
+# Note that the NuGetAuthenticate task should be called after SetupNugetSources.
+# This ensures that:
+# - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt)
+# - The credential provider is installed.
+#
+# This logic is also abstracted into enable-internal-sources.yml.
 
 [CmdletBinding()]
 param (
  [Parameter(Mandatory = $true)][string]$ConfigFile,
- [Parameter(Mandatory = $true)][string]$Password
+ $Password
 )
 
 $ErrorActionPreference = "Stop"
@@ -48,11 +48,17 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern
  else {
  Write-Host "Package source $SourceName already present."
  }
+
  AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd
 }
 
 # Add a credential node for the specified source
 function AddCredential($creds, $source, $username, $pwd) {
+ # If no cred supplied, don't do anything.
+ if (!$pwd) {
+ return;
+ }
+
  # Looks for credential configuration for the given SourceName. Create it if none is found.
  $sourceElement = $creds.SelectSingleNode($Source)
  if ($sourceElement -eq $null)
@@ -110,11 +116,6 @@ if (!(Test-Path $ConfigFile -PathType Leaf)) {
  ExitWithExitCode 1
 }
 
-if (!$Password) {
- Write-PipelineTelemetryError -Category 'Build' -Message 'Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Please supply a valid PAT'
- ExitWithExitCode 1
-}
-
 # Load NuGet.config
 $doc = New-Object System.Xml.XmlDocument
 $filename = (Get-Item $ConfigFile).FullName
@@ -127,11 +128,14 @@ if ($sources -eq $null) {
  $doc.DocumentElement.AppendChild($sources) | Out-Null
 }
 
-# Looks for a <PackageSourceCredentials> node. Create it if none is found.
-$creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials")
-if ($creds -eq $null) {
- $creds = $doc.CreateElement("packageSourceCredentials")
- $doc.DocumentElement.AppendChild($creds) | Out-Null
+$creds = $null
+if ($Password) {
+ # Looks for a <PackageSourceCredentials> node. Create it if none is found.
+ $creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials")
+ if ($creds -eq $null) {
+ $creds = $doc.CreateElement("packageSourceCredentials")
+ $doc.DocumentElement.AppendChild($creds) | Out-Null
+ }
 }
 
 # Check for disabledPackageSources; we'll enable any darc-int ones we find there
@@ -164,4 +168,4 @@ foreach ($dotnetVersion in $dotnetVersions) {
  }
 }
 
-$doc.Save($filename)
+$doc.Save($filename)

eng/common/SetupNugetSources.sh

@@ -1,28 +1,27 @@
 #!/usr/bin/env bash
 
-# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
-# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
+# This script adds internal feeds required to build commits that depend on internal package sources. For instance,
+# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables
+# disabled internal Maestro (darc-int*) feeds.
+# 
+# Optionally, this script also adds a credential entry for each of the internal feeds if supplied.
 #
-# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
-# under <packageSourceCredentials> for each Maestro's managed private feed. Two additional credential 
-# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
-#
-# This script needs to be called in every job that will restore packages and which the base repo has
-# private AzDO feeds in the NuGet.config.
-#
-# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
-# from the AzureDevOps-Artifact-Feeds-Pats variable group.
-#
-# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing.
+# See example call for this script below.
 #
 # - task: Bash@3
-# displayName: Setup Private Feeds Credentials
+# displayName: Setup Internal Feeds
 # inputs:
 # filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
-# arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+# arguments: $(Build.SourcesDirectory)/NuGet.config
 # condition: ne(variables['Agent.OS'], 'Windows_NT')
-# env:
-# Token: $(dn-bot-dnceng-artifact-feeds-rw)
+# - task: NuGetAuthenticate@1
+#
+# Note that the NuGetAuthenticate task should be called after SetupNugetSources.
+# This ensures that:
+# - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt)
+# - The credential provider is installed.
+#
+# This logic is also abstracted into enable-internal-sources.yml.
 
 ConfigFile=$1
 CredToken=$2
@@ -48,11 +47,6 @@ if [ ! -f "$ConfigFile" ]; then
  ExitWithExitCode 1
 fi
 
-if [ -z "$CredToken" ]; then
- Write-PipelineTelemetryError -category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Please supply a valid PAT"
- ExitWithExitCode 1
-fi
-
 if [[ `uname -s` == "Darwin" ]]; then
  NL=$'\\\n'
  TB=''
@@ -140,18 +134,20 @@ PackageSources+="$IFS"
 PackageSources+=$(grep -oh '"darc-int-[^"]*"' $ConfigFile | tr -d '"')
 IFS=$PrevIFS
 
-for FeedName in ${PackageSources[@]} ; do
- # Check if there is no existing credential for this FeedName
- grep -i "<$FeedName>" $ConfigFile 
- if [ "$?" != "0" ]; then
- echo "Adding credentials for $FeedName."
+if [ "$CredToken" ]; then
+ for FeedName in ${PackageSources[@]} ; do
+ # Check if there is no existing credential for this FeedName
+ grep -i "<$FeedName>" $ConfigFile 
+ if [ "$?" != "0" ]; then
+ echo "Adding credentials for $FeedName."
 
- PackageSourceCredentialsNodeFooter="</packageSourceCredentials>"
- NewCredential="${TB}${TB}<$FeedName>${NL}<add key=\"Username\" value=\"dn-bot\" />${NL}<add key=\"ClearTextPassword\" value=\"$CredToken\" />${NL}</$FeedName>"
+  PackageSourceCredentialsNodeFooter="</packageSourceCredentials>"
+  NewCredential="${TB}${TB}<$FeedName>${NL}<add key=\"Username\" value=\"dn-bot\" />${NL}<add key=\"ClearTextPassword\" value=\"$CredToken\" />${NL}</$FeedName>"
 
- sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile
- fi
-done
+ sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile
+ fi
+ done
+fi
 
 # Re-enable any entries in disabledPackageSources where the feed name contains darc-int
 grep -i "<disabledPackageSources>" $ConfigFile

eng/common/core-templates/job/job.yml

@@ -200,29 +200,28 @@ jobs:
  publishArtifacts: false
 
  # Publish test results
- - ${{ if and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')) }}:
- - ${{ if eq(parameters.testResultsFormat, 'xunit') }}:
- - task: PublishTestResults@2
- displayName: Publish XUnit Test Results
- inputs:
- testResultsFormat: 'xUnit'
- testResultsFiles: '*.xml'
- searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
- testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit
- mergeTestResults: ${{ parameters.mergeTestResults }}
- continueOnError: true
- condition: always()
- - ${{ if eq(parameters.testResultsFormat, 'vstest') }}:
- - task: PublishTestResults@2
- displayName: Publish TRX Test Results
- inputs:
- testResultsFormat: 'VSTest'
- testResultsFiles: '*.trx'
- searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
- testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx
- mergeTestResults: ${{ parameters.mergeTestResults }}
- continueOnError: true
- condition: always()
+ - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}:
+ - task: PublishTestResults@2
+ displayName: Publish XUnit Test Results
+ inputs:
+ testResultsFormat: 'xUnit'
+ testResultsFiles: '*.xml'
+ searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
+ testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit
+ mergeTestResults: ${{ parameters.mergeTestResults }}
+ continueOnError: true
+ condition: always()
+ - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}:
+ - task: PublishTestResults@2
+ displayName: Publish TRX Test Results
+ inputs:
+ testResultsFormat: 'VSTest'
+ testResultsFiles: '*.trx'
+ searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
+ testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx
+ mergeTestResults: ${{ parameters.mergeTestResults }}
+ continueOnError: true
+ condition: always()
 
  # gather artifacts
  - ${{ if ne(parameters.artifacts.publish, '') }}:
@@ -246,6 +245,8 @@ jobs:
  SourceFolder: 'artifacts/log'
  Contents: '**'
  TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/log'
+ continueOnError: true
+ condition: always()
 
  - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
  - task: CopyFiles@2
@@ -254,13 +255,16 @@ jobs:
  SourceFolder: 'artifacts/log/$(_BuildConfig)'
  Contents: '**'
  TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)'
+ continueOnError: true
+ condition: always()
  - ${{ if eq(parameters.enableBuildRetry, 'true') }}:
  - task: CopyFiles@2
  displayName: Gather buildconfiguration for build retry
  inputs:
  SourceFolder: '$(Build.SourcesDirectory)/eng/common/BuildConfiguration'
  Contents: '**'
  TargetFolder: '$(Build.ArtifactStagingDirectory)/eng/common/BuildConfiguration'
-
+ continueOnError: true
+ condition: always()
  - ${{ each step in parameters.artifactPublishSteps }}:
  - ${{ step }}

eng/common/core-templates/job/publish-build-assets.yml

@@ -87,13 +87,15 @@ jobs:
 
  - task: NuGetAuthenticate@1
 
- - task: PowerShell@2
+ - task: AzureCLI@2
  displayName: Publish Build Assets
  inputs:
- filePath: eng\common\sdk-task.ps1
+ azureSubscription: "Darc: Maestro Production"
+ scriptType: ps
+ scriptLocation: scriptPath
+ scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1
  arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
  /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
- /p:BuildAssetRegistryToken=$(MaestroAccessToken)
  /p:MaestroApiEndpoint=https://maestro.dot.net
  /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
  /p:OfficialBuildId=$(Build.BuildNumber)
@@ -153,14 +155,16 @@ jobs:
  PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
  is1ESPipeline: ${{ parameters.is1ESPipeline }}
 
- - task: PowerShell@2
+ - task: AzureCLI@2
  displayName: Publish Using Darc
  inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
+ azureSubscription: "Darc: Maestro Production"
+ scriptType: ps
+ scriptLocation: scriptPath
+ scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
  arguments: -BuildId $(BARBuildId) 
  -PublishingInfraVersion 3
  -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
- -MaestroToken '$(MaestroApiAccessToken)'
  -WaitPublishingFinish true
  -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
  -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'

eng/common/core-templates/job/source-build.yml

@@ -33,6 +33,12 @@ parameters:
 
  is1ESPipeline: ''
 
+ # If set to true and running on a non-public project,
+ # Internal nuget and blob storage locations will be enabled.
+ # This is not enabled by default because many repositories do not need internal sources
+ # and do not need to have the required service connections approved in the pipeline.
+ enableInternalSources: false
+
 jobs:
 - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
  displayName: Source-Build (${{ parameters.platform.name }})
@@ -74,6 +80,13 @@ jobs:
  - ${{ if eq(parameters.is1ESPipeline, '') }}:
  - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error
 
+ - ${{ if eq(parameters.enableInternalSources, true) }}:
+ - template: /eng/common/core-templates/steps/enable-internal-sources.yml
+ parameters:
+ is1ESPipeline: ${{ parameters.is1ESPipeline }}
+ - template: /eng/common/core-templates/steps/enable-internal-runtimes.yml
+ parameters:
+ is1ESPipeline: ${{ parameters.is1ESPipeline }}
  - template: /eng/common/core-templates/steps/source-build.yml
  parameters:
  is1ESPipeline: ${{ parameters.is1ESPipeline }}

eng/common/core-templates/job/source-index-stage1.yml

@@ -1,7 +1,7 @@
 parameters:
  runAsPublic: false
- sourceIndexUploadPackageVersion: 2.0.0-20240502.12
- sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2
+ sourceIndexUploadPackageVersion: 2.0.0-20240522.1
+ sourceIndexProcessBinlogPackageVersion: 1.0.1-20240522.1
  sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
  sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
  preSteps: []
@@ -88,4 +88,4 @@ jobs:
  displayName: "Login to Azure"
 
  - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
- displayName: Upload stage1 artifacts to source index
+ displayName: Upload stage1 artifacts to source index