Skip to content

Commit

Permalink
[main] Update dependencies from dotnet/arcade (#5118)
Browse files Browse the repository at this point in the history
* Update dependencies from https://github.com/dotnet/arcade build 20240621.4

Microsoft.SourceBuild.Intermediate.arcade , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.XliffTasks
 From Version 9.0.0-beta.24306.4 -> To Version 9.0.0-beta.24321.4

* Update dependencies from https://github.com/dotnet/arcade build 20240627.1

Microsoft.SourceBuild.Intermediate.arcade , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.XliffTasks
 From Version 9.0.0-beta.24306.4 -> To Version 9.0.0-beta.24327.1

* Update dependencies from https://github.com/dotnet/arcade build 20240627.1

Microsoft.SourceBuild.Intermediate.arcade , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.XliffTasks
 From Version 9.0.0-beta.24306.4 -> To Version 9.0.0-beta.24327.1

* Update dependencies from https://github.com/dotnet/arcade build 20240702.2

Microsoft.SourceBuild.Intermediate.arcade , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.XliffTasks
 From Version 9.0.0-beta.24306.4 -> To Version 9.0.0-beta.24352.2

* Add arcade issue workaround

* add arcade issue workaround

* add arcade issue workaround

---------

Co-authored-by: dotnet-maestro[bot] <dotnet-maestro[bot]@users.noreply.github.com>
Co-authored-by: Jakub Jareš <[email protected]>
  • Loading branch information
3 people authored Jul 10, 2024
1 parent 0a59896 commit 3a6c061
Show file tree
Hide file tree
Showing 47 changed files with 260 additions and 413 deletions.
6 changes: 5 additions & 1 deletion Directory.Build.targets
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
<?xml version="1.0" encoding="utf-8"?>
<Project>

<PropertyGroup>
<!-- Temporary workaround for Arcade issue in net9-preview5 -->
<_NetFrameworkHostedCompilersVersion Condition="'$(_NetFrameworkHostedCompilersVersion)' == ''">4.11.0-3.24280.3</_NetFrameworkHostedCompilersVersion>
</PropertyGroup>

<Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />

<!--
Expand Down
12 changes: 6 additions & 6 deletions eng/Version.Details.xml
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,14 @@
</Dependency>
</ProductDependencies>
<ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24306.4">
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24352.2">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>7507f80c8db285bbc9939c1dff522a761cf4edc0</Sha>
<Sha>4a7d983f833d6b86365ea1b2b4d6ee72fbdbf944</Sha>
</Dependency>
<!-- Intermediate is necessary for source build. -->
<Dependency Name="Microsoft.SourceBuild.Intermediate.arcade" Version="9.0.0-beta.24306.4">
<Dependency Name="Microsoft.SourceBuild.Intermediate.arcade" Version="9.0.0-beta.24352.2">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>7507f80c8db285bbc9939c1dff522a761cf4edc0</Sha>
<Sha>4a7d983f833d6b86365ea1b2b4d6ee72fbdbf944</Sha>
<SourceBuild RepoName="arcade" ManagedOnly="true" />
</Dependency>
<Dependency Name="Microsoft.DiaSymReader.Pdb2Pdb" Version="1.1.0-beta2-19575-01">
Expand All @@ -62,9 +62,9 @@
<Uri>https://github.com/dotnet/symreader-converter</Uri>
<Sha>c5ba7c88f92e2dde156c324a8c8edc04d9fa4fe0</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.24306.4">
<Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.24352.2">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>7507f80c8db285bbc9939c1dff522a761cf4edc0</Sha>
<Sha>4a7d983f833d6b86365ea1b2b4d6ee72fbdbf944</Sha>
</Dependency>
</ToolsetDependencies>
</Dependencies>
143 changes: 82 additions & 61 deletions eng/common/SetupNugetSources.ps1
Original file line number Diff line number Diff line change
@@ -1,32 +1,31 @@
# This script adds internal feeds required to build commits that depend on internal package sources. For instance,
# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables
# disabled internal Maestro (darc-int*) feeds.
#
# Optionally, this script also adds a credential entry for each of the internal feeds if supplied. This credential
# is added via the standard environment variable VSS_NUGET_EXTERNAL_FEED_ENDPOINTS. See
# https://github.com/microsoft/artifacts-credprovider/tree/v1.1.1?tab=readme-ov-file#environment-variables for more details
#
# Optionally, this script also adds a credential entry for each of the internal feeds if supplied.
#
# See example call for this script below.
#
# - task: PowerShell@2
# displayName: Setup Internal Feeds
# displayName: Setup Private Feeds Credentials
# condition: eq(variables['Agent.OS'], 'Windows_NT')
# inputs:
# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config
# - task: NuGetAuthenticate@1
#
# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
# env:
# Token: $(dn-bot-dnceng-artifact-feeds-rw)
#
# Note that the NuGetAuthenticate task should be called after SetupNugetSources.
# This ensures that:
# - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt)
# - The credential provider is installed
# - The credential provider is installed.
#
# This logic is also abstracted into enable-internal-sources.yml.

[CmdletBinding()]
param (
[Parameter(Mandatory = $true)][string]$ConfigFile,
[string]$Password
$Password
)

$ErrorActionPreference = "Stop"
Expand All @@ -35,23 +34,12 @@ Set-StrictMode -Version 2.0

. $PSScriptRoot\tools.ps1

$feedEndpoints = $null

# If a credential is provided, ensure that we don't overwrite the current set of
# credentials that may have been provided by a previous call to the credential provider.
if ($Password -and $null -ne $env:VSS_NUGET_EXTERNAL_FEED_ENDPOINTS) {
$feedEndpoints = $env:VSS_NUGET_EXTERNAL_FEED_ENDPOINTS | ConvertFrom-Json
} elseif ($Password) {
$feedEndpoints = @{ endpointCredentials = @() }
}

# Add source entry to PackageSources
function AddPackageSource($sources, $SourceName, $SourceEndPoint, $pwd) {
function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) {
$packageSource = $sources.SelectSingleNode("add[@key='$SourceName']")

if ($null -eq $packageSource)
if ($packageSource -eq $null)
{
Write-Host "`tAdding package source" $SourceName
$packageSource = $doc.CreateElement("add")
$packageSource.SetAttribute("key", $SourceName)
$packageSource.SetAttribute("value", $SourceEndPoint)
Expand All @@ -61,33 +49,63 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $pwd) {
Write-Host "Package source $SourceName already present."
}

if ($pwd) {
$feedEndpoints.endpointCredentials = AddCredential -endpointCredentials $feedEndpoints.endpointCredentials -source $SourceEndPoint -pwd $pwd
}
AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd
}

# Add a new feed endpoint credential
function AddCredential([array]$endpointCredentials, $source, $pwd) {
$endpointCredentials += @{
endpoint = $source;
password = $pwd
# Add a credential node for the specified source
function AddCredential($creds, $source, $username, $pwd) {
# If no cred supplied, don't do anything.
if (!$pwd) {
return;
}
return $endpointCredentials

# Looks for credential configuration for the given SourceName. Create it if none is found.
$sourceElement = $creds.SelectSingleNode($Source)
if ($sourceElement -eq $null)
{
$sourceElement = $doc.CreateElement($Source)
$creds.AppendChild($sourceElement) | Out-Null
}

# Add the <Username> node to the credential if none is found.
$usernameElement = $sourceElement.SelectSingleNode("add[@key='Username']")
if ($usernameElement -eq $null)
{
$usernameElement = $doc.CreateElement("add")
$usernameElement.SetAttribute("key", "Username")
$sourceElement.AppendChild($usernameElement) | Out-Null
}
$usernameElement.SetAttribute("value", $Username)

# Add the <ClearTextPassword> to the credential if none is found.
# Add it as a clear text because there is no support for encrypted ones in non-windows .Net SDKs.
# -> https://github.com/NuGet/Home/issues/5526
$passwordElement = $sourceElement.SelectSingleNode("add[@key='ClearTextPassword']")
if ($passwordElement -eq $null)
{
$passwordElement = $doc.CreateElement("add")
$passwordElement.SetAttribute("key", "ClearTextPassword")
$sourceElement.AppendChild($passwordElement) | Out-Null
}

$passwordElement.SetAttribute("value", $pwd)
}

function InsertMaestroInternalFeedCredentials($Sources, $pwd) {
$maestroInternalSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) {
$maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")

ForEach ($PackageSource in $maestroInternalSources) {
Write-Host "`tAdding credential for Maestro's feed:" $PackageSource.Key
$feedEndpoints.endpointCredentials = AddCredential -endpointCredentials $feedEndpoints.endpointCredentials -source $PackageSource.value -pwd $pwd
Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds."

ForEach ($PackageSource in $maestroPrivateSources) {
Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key
AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd
}
}

function EnableInternalPackageSources($DisabledPackageSources) {
$maestroInternalSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]")
ForEach ($DisabledPackageSource in $maestroInternalSources) {
Write-Host "`tEnsuring internal source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource"
function EnablePrivatePackageSources($DisabledPackageSources) {
$maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]")
ForEach ($DisabledPackageSource in $maestroPrivateSources) {
Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource"
# Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries
$DisabledPackageSources.RemoveChild($DisabledPackageSource)
}
Expand All @@ -105,46 +123,49 @@ $doc.Load($filename)

# Get reference to <PackageSources> or create one if none exist already
$sources = $doc.DocumentElement.SelectSingleNode("packageSources")
if ($null -eq $sources) {
if ($sources -eq $null) {
$sources = $doc.CreateElement("packageSources")
$doc.DocumentElement.AppendChild($sources) | Out-Null
}

$creds = $null
if ($Password) {
# Looks for a <PackageSourceCredentials> node. Create it if none is found.
$creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials")
if ($creds -eq $null) {
$creds = $doc.CreateElement("packageSourceCredentials")
$doc.DocumentElement.AppendChild($creds) | Out-Null
}
}

# Check for disabledPackageSources; we'll enable any darc-int ones we find there
$disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources")
if ($null -ne $disabledSources) {
if ($disabledSources -ne $null) {
Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node"
EnableInternalPackageSources -DisabledPackageSources $disabledSources
EnablePrivatePackageSources -DisabledPackageSources $disabledSources
}

if ($Password) {
InsertMaestroInternalFeedCredentials -Sources $sources -pwd $Password
}
$userName = "dn-bot"

# Insert credential nodes for Maestro's private feeds
InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password

# 3.1 uses a different feed url format so it's handled differently here
$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
if ($null -ne $dotnet31Source) {
AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json" -pwd $Password
AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json" -pwd $Password
if ($dotnet31Source -ne $null) {
AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password
AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
}

$dotnetVersions = @('5','6','7','8')

foreach ($dotnetVersion in $dotnetVersions) {
$feedPrefix = "dotnet" + $dotnetVersion;
$dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']")
if ($dotnetSource) {
AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedprefix-internal/nuget/v3/index.json" -pwd $Password
AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v3/index.json" -pwd $Password
if ($dotnetSource -ne $null) {
AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password
AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
}
}

$doc.Save($filename)

# If any credentials were added or altered, update the VSS_NUGET_EXTERNAL_FEED_ENDPOINTS environment variable
if ($null -ne $feedEndpoints) {
# ci is set to true so vso logging commands will be used.
$ci = $true
Write-PipelineSetVariable -Name 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' -Value $($feedEndpoints | ConvertTo-Json) -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED' -Value "False" -IsMultiJobVariable $false
}
2 changes: 1 addition & 1 deletion eng/common/SetupNugetSources.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/usr/bin/env bash

# This script adds internal feeds required to build commits that depend on intenral package sources. For instance,
# This script adds internal feeds required to build commits that depend on internal package sources. For instance,
# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables
# disabled internal Maestro (darc-int*) feeds.
#
Expand Down
24 changes: 3 additions & 21 deletions eng/common/core-templates/job/job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,11 @@ parameters:
enablePublishTestResults: false
enablePublishUsingPipelines: false
enableBuildRetry: false
disableComponentGovernance: ''
componentGovernanceIgnoreDirectories: ''
mergeTestResults: false
testRunTitle: ''
testResultsFormat: ''
name: ''
componentGovernanceSteps: []
preSteps: []
artifactPublishSteps: []
runAsPublic: false
Expand Down Expand Up @@ -170,17 +169,8 @@ jobs:
uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }}
continueOnError: true

- template: /eng/common/core-templates/steps/component-governance.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
${{ if eq(parameters.disableComponentGovernance, '') }}:
${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
disableComponentGovernance: false
${{ else }}:
disableComponentGovernance: true
${{ else }}:
disableComponentGovernance: ${{ parameters.disableComponentGovernance }}
componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }}
- ${{ each step in parameters.componentGovernanceSteps }}:
- ${{ step }}

- ${{ if eq(parameters.enableMicrobuild, 'true') }}:
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
Expand All @@ -190,14 +180,6 @@ jobs:
continueOnError: ${{ parameters.continueOnError }}
env:
TeamName: $(_TeamName)
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}:
- template: /eng/common/core-templates/steps/generate-sbom.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
PackageVersion: ${{ parameters.packageVersion}}
BuildDropPath: ${{ parameters.buildDropPath }}
IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }}
publishArtifacts: false

# Publish test results
- ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}:
Expand Down
16 changes: 10 additions & 6 deletions eng/common/core-templates/job/publish-build-assets.yml
Original file line number Diff line number Diff line change
Expand Up @@ -87,13 +87,15 @@ jobs:

- task: NuGetAuthenticate@1

- task: PowerShell@2
- task: AzureCLI@2
displayName: Publish Build Assets
inputs:
filePath: eng\common\sdk-task.ps1
azureSubscription: "Darc: Maestro Production"
scriptType: ps
scriptLocation: scriptPath
scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1
arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
/p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
/p:BuildAssetRegistryToken=$(MaestroAccessToken)
/p:MaestroApiEndpoint=https://maestro.dot.net
/p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
/p:OfficialBuildId=$(Build.BuildNumber)
Expand Down Expand Up @@ -153,14 +155,16 @@ jobs:
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
is1ESPipeline: ${{ parameters.is1ESPipeline }}

- task: PowerShell@2
- task: AzureCLI@2
displayName: Publish Using Darc
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
azureSubscription: "Darc: Maestro Production"
scriptType: ps
scriptLocation: scriptPath
scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
arguments: -BuildId $(BARBuildId)
-PublishingInfraVersion 3
-AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
-MaestroToken '$(MaestroApiAccessToken)'
-WaitPublishingFinish true
-ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
-SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
Expand Down
16 changes: 2 additions & 14 deletions eng/common/core-templates/job/source-index-stage1.yml
Original file line number Diff line number Diff line change
Expand Up @@ -69,23 +69,11 @@ jobs:

- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: AzureCLI@2
displayName: Get stage 1 auth token
displayName: Log in to Azure and upload stage1 artifacts to source index
inputs:
azureSubscription: 'SourceDotNet Stage1 Publish'
addSpnToEnvironment: true
scriptType: 'ps'
scriptLocation: 'inlineScript'
inlineScript: |
echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId"
echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken"
echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId"
- script: |
echo "Client ID: $(ARM_CLIENT_ID)"
echo "ID Token: $(ARM_ID_TOKEN)"
echo "Tenant ID: $(ARM_TENANT_ID)"
az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN)
displayName: "Login to Azure"
- script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
displayName: Upload stage1 artifacts to source index
$(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
2 changes: 0 additions & 2 deletions eng/common/core-templates/post-build/common-variables.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ variables:
# Default Maestro++ API Endpoint and API Version
- name: MaestroApiEndPoint
value: "https://maestro.dot.net"
- name: MaestroApiAccessToken
value: $(MaestroAccessToken)
- name: MaestroApiVersion
value: "2020-02-20"

Expand Down
Loading

0 comments on commit 3a6c061

Please sign in to comment.