Skip to content

Commit 7189b4a

Browse files
mmitchemmitche
authored
Revert "Streamline and simplify v3 publishing job (dotnet#8334)" (dotnet#8339)
This reverts commit 5de8a0a.
1 parent 5de8a0a commit 7189b4a

12 files changed

Lines changed: 578 additions & 199 deletions

‎eng/common/post-build/publish-using-darc.ps1‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,13 @@ param(
55
[Parameter(Mandatory=$true)][string] $MaestroToken,
66
[Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
77
[Parameter(Mandatory=$true)][string] $WaitPublishingFinish,
8+
[Parameter(Mandatory=$false)][string] $EnableSourceLinkValidation,
9+
[Parameter(Mandatory=$false)][string] $EnableSigningValidation,
10+
[Parameter(Mandatory=$false)][string] $EnableNugetValidation,
11+
[Parameter(Mandatory=$false)][string] $PublishInstallersAndChecksums,
812
[Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
9-
[Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters
13+
[Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters,
14+
[Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters
1015
)
1116

1217
try {

‎eng/common/templates/post-build/post-build.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -263,5 +263,6 @@ stages:
263263
-AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
264264
-MaestroToken '$(MaestroApiAccessToken)'
265265
-WaitPublishingFinish true
266+
-PublishInstallersAndChecksums ${{ parameters.publishInstallersAndChecksums }}
266267
-ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
267268
-SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'

‎eng/promote-build.yml‎

Lines changed: 107 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -17,20 +17,6 @@ parameters:
1717
type: string
1818
default: ' '
1919

20-
- name: SymbolPublishingAdditionalParameters
21-
displayName: Additional (MSBuild) properties for symbol publishing
22-
type: string
23-
default: ' '
24-
25-
- name: ArtifactsPublishingAdditionalParameters
26-
displayName: Additional (MSBuild) properties for general asset publishing
27-
type: string
28-
default: ' '
29-
30-
# The parameters below here are legacy. They are passed by add-build-to-channel
31-
# to the build pipeline, and if they are not present in the pipeline, then queueing
32-
# will fail. Remove once add-build-to-channel has been updated to remove the parameters.
33-
3420
- name: EnableSourceLinkValidation
3521
displayName: Should Sourcelink validation be performed?
3622
type: boolean
@@ -50,6 +36,16 @@ parameters:
5036
displayName: Should installers and checksums be published?
5137
type: boolean
5238
default: true
39+
40+
- name: SymbolPublishingAdditionalParameters
41+
displayName: Additional (MSBuild) properties for symbol publishing
42+
type: string
43+
default: ' '
44+
45+
- name: ArtifactsPublishingAdditionalParameters
46+
displayName: Additional (MSBuild) properties for general asset publishing
47+
type: string
48+
default: ' '
5349

5450
- name: SigningValidationAdditionalParameters
5551
displayName: Additional (MSBuild) properties for signing validation
@@ -58,10 +54,101 @@ parameters:
5854

5955
trigger: none
6056

57+
variables:
58+
_DotNetArtifactsCategory: .NETCore
59+
60+
pool:
61+
vmImage: ubuntu-latest
62+
6163
stages:
62-
- template: \eng\publishing\v3\publish.yml
63-
parameters:
64-
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
65-
BARBuildId: ${{ parameters.BARBuildId }}
66-
symbolPublishingAdditionalParameters: ${{ parameters.SymbolPublishingAdditionalParameters }}
67-
artifactsPublishingAdditionalParameters: ${{ parameters.ArtifactsPublishingAdditionalParameters }}
64+
- stage: prepare_promotion
65+
displayName: Prepare for Promotion
66+
jobs:
67+
- job:
68+
displayName: Validate Parameters
69+
variables:
70+
- template: common\templates\post-build\common-variables.yml
71+
steps:
72+
- checkout: none
73+
74+
- task: PowerShell@2
75+
displayName: Validate Build & Channel
76+
inputs:
77+
targetType: inline
78+
script: |
79+
# Keeping this script inline so that we don't need to checkout the whole repo to use just one file
80+
try {
81+
$buildApiEndpoint = "$(MaestroApiEndPoint)/api/builds/${Env:BARBuildId}?api-version=$(MaestroApiVersion)"
82+
83+
$apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]'
84+
$apiHeaders.Add('Accept', 'application/json')
85+
$apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}")
86+
87+
$buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
88+
89+
if (!$buildInfo) {
90+
Write-Host "Build with BAR ID ${Env:BARBuildId} was not found in BAR!"
91+
exit 1
92+
}
93+
94+
$channels = ${Env:PromoteToChannelIds} -split "-"
95+
foreach ($channelId in $channels) {
96+
$channelApiEndpoint = "$(MaestroApiEndPoint)/api/channels/${channelId}?api-version=$(MaestroApiVersion)"
97+
$channelInfo = try { Invoke-WebRequest -Method Get -Uri $channelApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
98+
99+
if (!$channelInfo) {
100+
Write-Host "Channel with ID ${channelId} was not found in BAR. Aborting."
101+
exit 1
102+
}
103+
}
104+
105+
$azureDevOpsBuildNumber = $buildInfo.azureDevOpsBuildNumber
106+
$channelName = $channelInfo.name
107+
$azureDevOpsRepository = "Unknown"
108+
$lastIndexOfSlash = $buildInfo.azureDevOpsRepository.LastIndexOf('/')
109+
110+
if ($lastIndexOfSlash -ne -1) {
111+
$azureDevOpsRepository = $buildInfo.azureDevOpsRepository.Substring($lastIndexOfSlash + 1)
112+
113+
# Invalid chars in Azdo build number: '"', '/', ':', '<', '>', '\', '|', '?', '@', and '*'
114+
$azureDevOpsRepository = $azureDevOpsRepository -replace '["/:<>\\|?@*"]', '_'
115+
}
116+
117+
$buildNumberName = "Promoting $azureDevOpsRepository build $azureDevOpsBuildNumber to channel(s) ${Env:PromoteToChannelIds}#"
118+
119+
# Maximum buildnumber length is 255 chars
120+
if ($buildNumberName.Length -GT 255) {
121+
$buildNumberName = $buildNumberName.Substring(0, 255)
122+
}
123+
124+
Write-Host "##vso[build.updatebuildnumber]$buildNumberName"
125+
Write-Host "##vso[build.addbuildtag]$channelName"
126+
}
127+
catch {
128+
Write-Host $_
129+
Write-Host $_.Exception
130+
Write-Host $_.ScriptStackTrace
131+
exit 1
132+
}
133+
env:
134+
MAESTRO_API_TOKEN: $(MaestroApiAccessToken)
135+
BARBuildId: ${{ parameters.BARBuildId }}
136+
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
137+
138+
- ${{ if ge(parameters.PublishingInfraVersion, 3) }}:
139+
- template: \eng\publishing\v3\publish.yml
140+
parameters:
141+
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
142+
BARBuildId: ${{ parameters.BARBuildId }}
143+
144+
enableSourceLinkValidation: ${{ parameters.EnableSourceLinkValidation }}
145+
enableNugetValidation: ${{ parameters.EnableNugetValidation }}
146+
enableSigningValidation: ${{ parameters.EnableSigningValidation }}
147+
148+
validateDependsOn:
149+
- prepare_promotion
150+
151+
publishInstallersAndChecksums: ${{ parameters.PublishInstallersAndChecksums }}
152+
symbolPublishingAdditionalParameters: ${{ parameters.SymbolPublishingAdditionalParameters }}
153+
artifactsPublishingAdditionalParameters: ${{ parameters.ArtifactsPublishingAdditionalParameters }}
154+
signingValidationAdditionalParameters: ${{ parameters.SigningValidationAdditionalParameters }}

‎eng/publishing/v3/common-variables.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
variables:
2+
- group: AzureDevOps-Artifact-Feeds-Pats
3+
- group: DotNet-Blob-Feed
4+
- group: DotNet-DotNetCli-Storage
5+
- group: DotNet-MSRC-Storage
6+
- group: Publish-Build-Assets
7+
8+
# Default Maestro++ API Endpoint and API Version
9+
- name: MaestroApiEndPoint
10+
value: "https://maestro-prod.westus2.cloudapp.azure.com"
11+
- name: MaestroApiAccessToken
12+
value: $(MaestroAccessToken)
13+
- name: MaestroApiVersion
14+
value: "2020-02-20"
15+
16+
- name: SourceLinkCLIVersion
17+
value: 3.0.0
18+
19+
# Skip component governance and codesign validation for SDL. These jobs
20+
# create no content.
21+
- name: skipComponentGovernanceDetection
22+
value: true
23+
- name: runCodesignValidationInjection
24+
value: false

‎eng/publishing/v3/nuget-validation.yml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
jobs:
2+
- job:
3+
displayName: NuGet Validation
4+
dependsOn: setupMaestroVars
5+
pool:
6+
vmImage: 'windows-2019'
7+
variables:
8+
- name: AzDOProjectName
9+
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
10+
- name: AzDOPipelineId
11+
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
12+
- name: AzDOBuildId
13+
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
14+
steps:
15+
- task: DownloadBuildArtifacts@0
16+
displayName: Download Package Artifacts
17+
inputs:
18+
buildType: specific
19+
buildVersionToDownload: specific
20+
project: $(AzDOProjectName)
21+
pipeline: $(AzDOPipelineId)
22+
buildId: $(AzDOBuildId)
23+
artifactName: PackageArtifacts
24+
checkDownloadedFiles: true
25+
26+
- task: PowerShell@2
27+
displayName: Validate
28+
inputs:
29+
targetType: inline
30+
script: |
31+
try {
32+
$ErrorActionPreference = 'Stop'
33+
Set-StrictMode -Version 2.0
34+
35+
# `tools.ps1` requires $ci to be $true
36+
$ci = $true
37+
$disableConfigureToolsetImport = $true
38+
. ${Env:BUILD_SOURCESDIRECTORY}\eng\common\tools.ps1
39+
40+
$ToolDestinationPath = "${Env:AGENT_BUILDDIRECTORY}\Extract\"
41+
$PackagesPath = "${Env:BUILD_ARTIFACTSTAGINGDIRECTORY}\PackageArtifacts\"
42+
$url = 'https://raw.githubusercontent.com/NuGet/NuGetGallery/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1'
43+
44+
New-Item -ItemType 'directory' -Path ${ToolDestinationPath} -Force
45+
46+
Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1
47+
48+
& ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg
49+
}
50+
catch {
51+
Write-Host $_.ScriptStackTrace
52+
Write-PipelineTelemetryError -Category 'NuGetValidation' -Message $_
53+
ExitWithExitCode 1
54+
}

‎eng/publishing/v3/postbuild-checks.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
jobs:
2+
- job:
3+
displayName: Post-build Checks
4+
dependsOn: setupMaestroVars
5+
variables:
6+
- name: TargetChannels
7+
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
8+
pool:
9+
vmImage: 'windows-2019'

‎eng/publishing/v3/publish-assets.yml‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
parameters:
2+
artifactsPublishingAdditionalParameters: ''
3+
publishInstallersAndChecksums: true
4+
PromoteToChannelIds: ''
5+
symbolPublishingAdditionalParameters: ''
6+
buildQuality: 'daily'
7+
8+
jobs:
9+
- job: publish_assets
10+
displayName: Publish Assets and Symbols
11+
dependsOn: setupMaestroVars
12+
timeoutInMinutes: 120
13+
variables:
14+
- group: DotNet-Symbol-Server-Pats
15+
- group: DotNetBuilds storage account tokens
16+
- name: BARBuildId
17+
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
18+
- name: IsStableBuild
19+
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
20+
- name: AzDOProjectName
21+
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
22+
- name: AzDOPipelineId
23+
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
24+
- name: AzDOBuildId
25+
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
26+
- name: AzDOAccount
27+
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildAccount'] ]
28+
29+
pool:
30+
vmImage: 'windows-2019'
31+
steps:
32+
- task: DownloadBuildArtifacts@0
33+
displayName: Download Build Assets
34+
continueOnError: true
35+
enabled: true
36+
inputs:
37+
buildType: specific
38+
buildVersionToDownload: specific
39+
project: $(AzDOProjectName)
40+
pipeline: $(AzDOPipelineId)
41+
buildId: $(AzDOBuildId)
42+
downloadType: 'specific'
43+
itemPattern: |
44+
AssetManifests/**
45+
BlobArtifacts/MergedManifest.xml
46+
PdbArtifacts/**
47+
ReleaseConfigs/SymbolPublishingExclusionsFile.txt
48+
downloadPath: '$(Build.ArtifactStagingDirectory)'
49+
50+
- task: NuGetToolInstaller@1
51+
displayName: 'Install NuGet.exe'
52+
53+
# This is necessary whenever we want to publish/restore to an AzDO private feed
54+
- task: NuGetAuthenticate@0
55+
displayName: 'Authenticate to AzDO Feeds'
56+
57+
- task: PowerShell@2
58+
displayName: Enable cross-org publishing
59+
inputs:
60+
filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1
61+
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
62+
63+
- task: PowerShell@2
64+
displayName: Publish packages, blobs and symbols
65+
inputs:
66+
filePath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1
67+
arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
68+
/p:PublishingInfraVersion=3
69+
/p:BARBuildId=$(BARBuildId)
70+
/p:TargetChannels='${{ parameters.PromoteToChannelIds }}'
71+
/p:IsInternalBuild=${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }}
72+
/p:NugetPath=$(NuGetExeToolPath)
73+
/p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
74+
/p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
75+
/p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
76+
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
77+
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
78+
/p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
79+
/p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
80+
/p:InternalInstallersAzureAccountKey=$(dotnetclimsrc-access-key)
81+
/p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
82+
/p:InternalChecksumsAzureAccountKey=$(dotnetclichecksumsmsrc-storage-key)
83+
/p:AzureDevOpsFeedsKey='$(dn-bot-dnceng-artifact-feeds-rw)'
84+
/p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
85+
/p:AkaMSClientId=$(akams-client-id)
86+
/p:AkaMSClientSecret=$(akams-client-secret)
87+
${{ parameters.artifactsPublishingAdditionalParameters }}
88+
/p:PDBArtifactsBasePath='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
89+
/p:SymbolPublishingExclusionsFile='$(Build.ArtifactStagingDirectory)/ReleaseConfigs/SymbolPublishingExclusionsFile.txt'
90+
${{ parameters.symbolPublishingAdditionalParameters}}
91+
/p:MsdlToken=$(microsoft-symbol-server-pat)
92+
/p:SymWebToken=$(symweb-symbol-server-pat)
93+
/p:BuildQuality='${{ parameters.buildQuality }}'
94+
/p:AzdoApiToken='$(dn-bot-all-orgs-build-rw-code-rw)'
95+
/p:ArtifactsBasePath='$(Build.ArtifactStagingDirectory)/'
96+
/p:BuildId='$(AzDOBuildId)'
97+
/p:AzureDevOpsOrg='$(AzDOAccount)'
98+
/p:AzureProject='$(AzDOProjectName)'
99+
/p:UseStreamingPublishing='true'
100+
/p:StreamingPublishingMaxClients=16
101+
/p:NonStreamingPublishingMaxClients=12
102+
/p:DotNetBuildsPublicUriBase64='$(dotnetbuilds-public-container-uri-base64)'
103+
/p:DotNetBuildsPublicChecksumsUriBase64='$(dotnetbuilds-public-container-checksum-uri-base64)'
104+
/p:DotNetBuildsInternalUriBase64='$(dotnetbuilds-internal-container-uri-base64)'
105+
/p:DotNetBuildsInternalChecksumsUriBase64='$(dotnetbuilds-internal-container-checksum-uri-base64)'
106+
- template: /eng/common/templates/steps/publish-logs.yml
107+
parameters:
108+
StageLabel: '${{ parameters.stageName }}'
109+
JobLabel: 'AssetsPublishing'

0 commit comments

Comments
 (0)