forked from dotnet/machinelearning
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathvsts-ci.yml
328 lines (306 loc) · 13.6 KB
/
vsts-ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
################################################################################
# ML.NET's official, signed build
################################################################################
variables:
- name: BuildConfig
value: Release
- name: _BuildConfig
value: $(BuildConfig)
- name: OfficialBuildId
value: $(BUILD.BUILDNUMBER)
- name: DOTNET_CLI_TELEMETRY_OPTOUT
value: 1
- name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE
value: 1
- name: DOTNET_MULTILEVEL_LOOKUP
value: 0
- name: Codeql.Enabled
value: true
- name: Codeql.SkipTaskAutoInjection
value: True #default to not inject CodeQL tasks, we'll enable it in a single job.
# CodeQL3000 needs this plumbed along as a variable to enable TSA.
- name: Codeql.TSAEnabled
value: true
# Default expects tsaoptions.json under SourceRoot.
- name: Codeql.TSAOptionsPath
value: '$(Build.SourcesDirectory)/.config/tsaoptions.json'
- name: LinuxImage
value: 1es-ubuntu-2204
- name: WindowsImage
value: 1es-windows-2019
- name: MacImage
value: macOS-13
- ${{ if and(notin(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}:
- name: enableSourceIndex
value: true
- template: /eng/common/templates-official/variables/pool-providers.yml@self
resources:
repositories:
- repository: 1ESPipelineTemplates
type: git
name: 1ESPipelineTemplates/1ESPipelineTemplates
ref: refs/tags/release
extends:
template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates
parameters:
featureFlags:
autoBaseline: true
sdl:
binskim:
enabled: true
scanOutputDirectoryOnly: true
policheck:
enabled: true
tsa:
enabled: true
pool:
name: $(DncEngInternalBuildPool)
image: $(WindowsImage)
os: windows
containers:
CentosContainer:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8-mlnet
UbuntuCrossArmContainer:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-mlnet-cross-arm
UbuntuCrossArm64Container:
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-mlnet-cross-arm64
stages:
- stage: build
displayName: Build
jobs:
################################################################################
- job: Linux_x64
################################################################################
pool:
name: $(DncEngInternalBuildPool)
image: $(LinuxImage)
os: linux
container: CentosContainer
templateContext:
outputs:
- output: buildArtifacts
PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets
ArtifactName: pkgassets
steps:
# Only build native assets to avoid conflicts.
- script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=x64 /p:CopyPackageAssets=true
displayName: Build Native Assets
################################################################################
- job: Linux_arm
################################################################################
variables:
ROOTFS_DIR: '/crossrootfs/arm'
pool:
name: $(DncEngInternalBuildPool)
image: $(LinuxImage)
os: linux
container: UbuntuCrossArmContainer
templateContext:
outputs:
- output: buildArtifacts
PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets
ArtifactName: pkgassets
steps:
# Only build native assets to avoid conflicts.
- script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=arm /p:CopyPackageAssets=true
displayName: Build Native Assets
################################################################################
- job: Linux_arm64
################################################################################
variables:
ROOTFS_DIR: '/crossrootfs/arm64'
pool:
name: $(DncEngInternalBuildPool)
image: $(LinuxImage)
os: linux
container: UbuntuCrossArm64Container
templateContext:
outputs:
- output: buildArtifacts
PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets
ArtifactName: pkgassets
steps:
# Only build native assets to avoid conflicts.
- script: ./build.sh -configuration $(BuildConfig) -projects $(Build.SourcesDirectory)/src/Native/Native.proj /p:TargetArchitecture=arm64 /p:CopyPackageAssets=true
displayName: Build Native Assets
################################################################################
- job: MacOS
################################################################################
pool:
name: Azure Pipelines
vmImage: $(MacImage)
os: macOS
templateContext:
outputs:
- output: buildArtifacts
PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets
ArtifactName: pkgassets
steps:
- script: export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 && rm '/usr/local/bin/2to3-3.11' && brew install $(Build.SourcesDirectory)/build/libomp.rb --build-from-source --formula
displayName: Install build dependencies
# Only build native assets to avoid conflicts.
- script: ./build.sh -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:CopyPackageAssets=true
displayName: Build Native Assets
################################################################################
- job: MacOS_Apple_Silicon
################################################################################
pool:
name: Azure Pipelines
vmImage: $(MacImage)
os: macOS
templateContext:
outputs:
- output: buildArtifacts
PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets
ArtifactName: pkgassets
steps:
# Work around MacOS Homebrew image/environment bug: https://github.com/actions/virtual-environments/issues/2322#issuecomment-749211076
- script: |
rm -rf /usr/local/bin/2to3
displayName: MacOS Homebrew bug Workaround
continueOnError: true
- script: brew update && brew install -f --overwrite [email protected] && brew install libomp && brew link libomp --force
displayName: Install build dependencies
# Only build native assets to avoid conflicts.
- script: ./build.sh -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=arm64 /p:CopyPackageAssets=true
displayName: Build Native Assets
################################################################################
- job: Windows_arm64
################################################################################
pool:
name: $(DncEngInternalBuildPool)
image: $(WindowsImage)
os: windows
templateContext:
outputs:
- output: buildArtifacts
PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets
ArtifactName: pkgassets
steps:
# Only build native assets to avoid conflicts.
- script: ./build.cmd -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=arm64 /p:CopyPackageAssets=true
displayName: Build Native Assets
# Terminate all dotnet build processes.
- script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown
displayName: Dotnet Server Shutdown
################################################################################
- job: Windows_x86
################################################################################
variables:
_TeamName: DotNetCore
pool:
name: $(DncEngInternalBuildPool)
image: $(WindowsImage)
os: windows
templateContext:
outputs:
- output: buildArtifacts
PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets
ArtifactName: pkgassets
steps:
# Only build native assets to avoid conflicts.
- script: ./build.cmd -projects $(Build.SourcesDirectory)/src/Native/Native.proj -configuration $(BuildConfig) /p:TargetArchitecture=x86 /p:CopyPackageAssets=true
displayName: Build Native Assets
# Terminate all dotnet build processes.
- script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown
displayName: Dotnet Server Shutdown
################################################################################
- job: Windows_x64
################################################################################
variables:
Codeql.SkipTaskAutoInjection: False # run CodeQL in this job
_TeamName: DotNetCore
pool:
name: $(DncEngInternalBuildPool)
image: $(WindowsImage)
os: windows
templateContext:
outputs:
- output: buildArtifacts
PathtoPublish: $(Build.SourcesDirectory)/artifacts/pkgassets
ArtifactName: pkgassets
steps:
# Build both native and managed assets.
- script: ./build.cmd -configuration $(BuildConfig) /p:TargetArchitecture=x64 /p:CopyPackageAssets=true
displayName: Build
- task: ComponentGovernanceComponentDetection@0
inputs:
scanType: 'Register'
verbosity: 'Verbose'
alertWarningLevel: 'High'
# Terminate all dotnet build processes.
- script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown
displayName: Dotnet Server Shutdown
################################################################################
- template: /eng/common/templates-official/job/job.yml@self
parameters:
name: Package
################################################################################
artifacts:
publish:
artifacts: false
logs: true
manifests: true
enableMicrobuild: true
enablePublishUsingPipelines: true
enableSourceIndex: true
dependsOn:
- Linux_x64
- Linux_arm
- Linux_arm64
- MacOS
- Windows_x86
- Windows_x64
- MacOS_Apple_Silicon
variables:
- DotnetVersionKind: $[variables.VERSIONKIND] # If no "VERSIONKIND" variable is set when queuing the publishing task, this defaults to empty string.
- NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages
- _SignType: real
- _TeamName: DotNetCore
- _InternalBuildArgs: /p:DotNetSignType=$(_SignType)
/p:TeamName=$(_TeamName)
/p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
/p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
/p:OfficialBuildId=$(OfficialBuildId)
/p:DotNetPublishUsingPipelines=$true
pool:
name: $(DncEngInternalBuildPool)
image: $(WindowsImage)
os: windows
steps:
# Download all agent packages from all previous phases
- task: DownloadBuildArtifacts@0
displayName: Download package assets
inputs:
artifactName: pkgassets
downloadPath: $(Build.SourcesDirectory)/artifacts
# Depending on the value of DotNetFinalVersionKind, the name of the package will change.
# For our nightly builds we want it to be empty, and when creating the official nugets, we want it to be "release"
# the value of the version kind is set when queuing the publishing job on AzureDevOps by adding a VERSIONKIND variable
# See more info in: https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Versioning.md#package-version
- script: ./build.cmd -configuration $(BuildConfig) -pack -sign -publish -ci /p:DotNetFinalVersionKind=$(DotnetVersionKind) $(_InternalBuildArgs)
displayName: Build Packages
# Terminate all dotnet build processes.
- script: $(Build.SourcesDirectory)/.dotnet/dotnet.exe build-server shutdown
displayName: Dotnet Server Shutdown
- ${{ if eq(variables.enableSourceIndex, 'true') }}:
- template: /eng/common/templates-official/job/source-index-stage1.yml@self
parameters:
sourceIndexBuildCommand: ./build.cmd -ci -configuration Release /p:TargetArchitecture=x64 /bl
binlogPath: msbuild.binlog
pool:
name: $(DncEngInternalBuildPool)
demands: ImageOverride -equals 1es-windows-2022
- template: /eng/common/templates-official/job/publish-build-assets.yml@self
parameters:
dependsOn: Package
enablePublishBuildArtifacts: true
publishAssetsImmediately: true
publishUsingPipelines: true
pool:
name: $(DncEngInternalBuildPool)
image: $(WindowsImage)
os: windows
- template: /eng/common/templates-official/post-build/post-build.yml@self
parameters:
publishingInfraVersion: 3