Skip to content

Commit

Permalink
Add the ability to do full rebuilds with updated copies of vcpkg-tool. (
Browse files Browse the repository at this point in the history
microsoft#24611)

* Add the ability to do full rebuilds with updated copies of vcpkg-tool.

Resurrection of microsoft#23753

* Extract all scripts out of inline blocks.

* Replicate all changes to the "unstable" testing.

* Fix bootstrap :)

* Fix version files validation path.

* Malformed.

* Paths and permissions.

* Fixed file name typo

* Update scripts/azure-pipelines/bootstrap-from-source.sh

Co-authored-by: Robert Schumacher <[email protected]>

Co-authored-by: Robert Schumacher <[email protected]>
  • Loading branch information
BillyONeal and ras0219-msft authored May 23, 2022
1 parent 413c11d commit 4c82b77
Show file tree
Hide file tree
Showing 9 changed files with 118 additions and 92 deletions.
15 changes: 15 additions & 0 deletions scripts/azure-pipelines/azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,53 +6,68 @@ variables:
linux-pool: 'PrLin-2022-05-19'
osx-pool: 'PrOsx-2022-02-04'

parameters:
- name: vcpkgToolSha
displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap'
type: string
default: 'use default'

jobs:
- template: windows/azure-pipelines.yml
parameters:
triplet: x86-windows
jobName: x86_windows
poolName: $(windows-pool)
vcpkgToolSha: ${{ parameters.vcpkgToolSha }}

- template: windows/azure-pipelines.yml
parameters:
triplet: x64-windows
jobName: x64_windows
poolName: $(windows-pool)
vcpkgToolSha: ${{ parameters.vcpkgToolSha }}

- template: windows/azure-pipelines.yml
parameters:
triplet: x64-windows-static
jobName: x64_windows_static
poolName: $(windows-pool)
vcpkgToolSha: ${{ parameters.vcpkgToolSha }}

- template: windows/azure-pipelines.yml
parameters:
triplet: x64-windows-static-md
jobName: x64_windows_static_md
poolName: $(windows-pool)
vcpkgToolSha: ${{ parameters.vcpkgToolSha }}

- template: windows/azure-pipelines.yml
parameters:
triplet: x64-uwp
jobName: x64_uwp
poolName: $(windows-pool)
vcpkgToolSha: ${{ parameters.vcpkgToolSha }}

- template: windows/azure-pipelines.yml
parameters:
triplet: arm64-windows
jobName: arm64_windows
poolName: $(windows-pool)
vcpkgToolSha: ${{ parameters.vcpkgToolSha }}

- template: windows/azure-pipelines.yml
parameters:
triplet: arm-uwp
jobName: arm_uwp
poolName: $(windows-pool)
vcpkgToolSha: ${{ parameters.vcpkgToolSha }}

- template: osx/azure-pipelines.yml
parameters:
poolName: $(osx-pool)
vcpkgToolSha: ${{ parameters.vcpkgToolSha }}

- template: linux/azure-pipelines.yml
parameters:
poolName: $(linux-pool)
vcpkgToolSha: ${{ parameters.vcpkgToolSha }}
9 changes: 9 additions & 0 deletions scripts/azure-pipelines/bootstrap-from-source.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#!/bin/sh
set -e

git clone https://github.com/microsoft/vcpkg-tool vcpkg-tool
git -C vcpkg-tool switch -d $1
rm -rf build.x64.release
cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=OFF -DVCPKG_WARNINGS_AS_ERRORS=OFF -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=OFF -B build.x64.release -S vcpkg-tool
ninja -C build.x64.release
mv build.x64.release/vcpkg vcpkg
32 changes: 16 additions & 16 deletions scripts/azure-pipelines/linux/azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,14 @@
# SPDX-License-Identifier: MIT
#

parameters:
- name: vcpkgToolSha
displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap'
type: string
default: 'use default'
- name: poolName
type: string

jobs:
- job: x64_linux
pool:
Expand All @@ -22,35 +30,27 @@ jobs:
value: "x-azblob,$(root-bin-url-ea),$(sas-bin-ea)"

steps:
- bash: df -h
displayName: 'Report on Disk Space'
# Note: /mnt is the Azure machines' temporary disk.
- bash: |
sudo mkdir /home/agent -m=777
sudo chown `id -u` /home/agent
exit 0
displayName: 'Create /home/agent'
# Note: /mnt is the Azure machines' temporary disk.
- bash: |
sudo rm -rf ${{ variables.VCPKG_DOWNLOADS }}
sudo mkdir ${{ variables.WORKING_ROOT }} -m=777
sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} -m=777
exit 0
displayName: 'Create ${{ variables.WORKING_ROOT }} and ${{ variables.VCPKG_DOWNLOADS }}'
- task: Bash@3
displayName: 'Create working directories'
- bash: ./bootstrap-vcpkg.sh
displayName: 'Bootstrap vcpkg'
inputs:
filePath: bootstrap-vcpkg.sh
condition: eq('use default', '${{ parameters.vcpkgToolSha }}')
- bash: ./scripts/azure-pipelines/bootstrap-from-source.sh ${{ parameters.vcpkgToolSha }}
displayName: "Build vcpkg with CMake"
condition: ne('use default', '${{ parameters.vcpkgToolSha }}')
- task: PowerShell@2
displayName: '*** Test Modified Ports for x64-linux'
displayName: '*** Test Modified Ports'
inputs:
failOnStderr: true
filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -BinarySourceStub "$(X_VCPKG_BINARY_SOURCE_STUB)" -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'
pwsh: true
- bash: |
df -h
displayName: 'Report on Disk Space After Build'
condition: always()
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: failure logs for x64-linux'
inputs:
Expand Down
30 changes: 15 additions & 15 deletions scripts/azure-pipelines/osx/azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,12 @@
#

parameters:
- name: poolName
type: string
- name: vcpkgToolSha
displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap'
type: string
default: 'use default'
- name: poolName
type: string

jobs:
- job: x64_osx
Expand All @@ -26,20 +30,21 @@ jobs:
value: "x-azurl,$(root-url-ea),$(sas-ea),readwrite"

steps:
- bash: df -h
displayName: 'Report on Disk Space'
- bash: |
sudo rm -rf ${{ variables.VCPKG_DOWNLOADS }} || 0
sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} || 0
sudo chmod 777 ${{ variables.VCPKG_DOWNLOADS }} || 0
exit 0
displayName: 'Create ${{ variables.VCPKG_DOWNLOADS }}'
- task: Bash@3
- bash: ./bootstrap-vcpkg.sh
displayName: 'Bootstrap vcpkg'
inputs:
filePath: bootstrap-vcpkg.sh
condition: eq('use default', '${{ parameters.vcpkgToolSha }}')
- bash: |
brew install cmake
./scripts/azure-pipelines/bootstrap-from-source.sh ${{ parameters.vcpkgToolSha }}
displayName: "Build vcpkg with CMake"
condition: ne('use default', '${{ parameters.vcpkgToolSha }}')
- task: PowerShell@2
displayName: '*** Test Modified Ports for x64-osx'
displayName: '*** Test Modified Ports'
inputs:
failOnStderr: true
filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
Expand All @@ -50,18 +55,13 @@ jobs:
-WorkingRoot "${{ variables.WORKING_ROOT }}"
-ArtifactStagingDirectory "$(Build.ArtifactStagingDirectory)"
pwsh: true
- bash: |
df -h
displayName: 'Report on Disk Space After Build'
condition: always()
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: failure logs for x64-osx'
inputs:
PathtoPublish: '$(Build.ArtifactStagingDirectory)/failure-logs'
ArtifactName: 'failure logs for x64-osx${{ variables.Postfix }}'
condition: ne(variables['FAILURE_LOGS_EMPTY'], 'True')
- bash: |
python3 scripts/file_script.py /Users/vagrant/Data/installed/vcpkg/info/
- bash: python3 scripts/file_script.py /Users/vagrant/Data/installed/vcpkg/info/
displayName: 'Build a file list for all packages'
condition: always()
- task: PublishBuildArtifacts@1
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
{
"$schema": "./vagrant-box-configuration.schema.json",
"brew": [
"autoconf",
"autoconf-archive",
"autoconf",
"automake",
"bison",
"cmake",
"gettext",
"gfortran",
"gperf",
Expand All @@ -13,6 +14,7 @@
"meson",
"mono",
"nasm",
"ninja",
"pkg-config",
"texinfo",
"yasm"
Expand Down
20 changes: 4 additions & 16 deletions scripts/azure-pipelines/windows-unstable/job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ jobs:
name: $(unstable-pool)
workspace:
clean: resources
timeoutInMinutes: 1440 # 1 day
timeoutInMinutes: 2880 # 2 days
variables:
- name: WORKING_ROOT
value: D:\
Expand Down Expand Up @@ -59,30 +59,18 @@ jobs:
filePath: 'scripts/azure-pipelines/windows-unstable/rearrange-msvc-drop-layout.ps1'
arguments: '-DropRoot "D:\msvc-drops\$(DropBuildNumber)" -BuildType ret'
pwsh: true
- task: PowerShell@2
displayName: 'Report on Disk Space'
condition: always()
inputs:
filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'
pwsh: true
- script: .\bootstrap-vcpkg.bat
displayName: 'Build vcpkg'
displayName: 'Bootstrap vcpkg'
- task: PowerShell@2
displayName: '*** Test Modified Ports and Prepare Test Logs ***'
displayName: '*** Test Modified Ports'
inputs:
failOnStderr: true
filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) -PassingIsPassing'
pwsh: true
- task: PowerShell@2
displayName: 'Report on Disk Space After Build'
condition: always()
inputs:
filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'
pwsh: true
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: failure logs for ${{ parameters.triplet }}'
inputs:
PathtoPublish: '$(Build.ArtifactStagingDirectory)\failure-logs'
ArtifactName: 'failure logs for ${{ parameters.triplet }}'
condition: always()
condition: ne(variables['FAILURE_LOGS_EMPTY'], 'True')
64 changes: 20 additions & 44 deletions scripts/azure-pipelines/windows/azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,20 @@
# SPDX-License-Identifier: MIT
#

parameters:
- name: vcpkgToolSha
displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap'
type: string
default: 'use default'
- name: triplet
type: string
default: 'x86-windows'
- name: jobName
type: string
default: 'x86_windows'
- name: poolName
type: string

jobs:
- job: ${{ parameters.jobName }}
pool:
Expand All @@ -26,14 +40,12 @@ jobs:
value: "x-azblob,$(root-bin-url-ea),$(sas-bin-ea)"

steps:
- task: PowerShell@2
displayName: 'Report on Disk Space'
condition: always()
inputs:
filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'
pwsh: true
- script: .\bootstrap-vcpkg.bat
displayName: 'Bootstrap vcpkg'
condition: eq('use default', '${{ parameters.vcpkgToolSha }}')
- script: .\scripts\azure-pipelines\windows\bootstrap-from-source.cmd ${{ parameters.vcpkgToolSha }}
displayName: "Build vcpkg with CMake"
condition: ne('use default', '${{ parameters.vcpkgToolSha }}')
# Note that we run docs checks before PR checks because they are likely to invalidate a whole run anyway
- task: Powershell@2
displayName: 'Generate Documentation'
Expand All @@ -59,7 +71,7 @@ jobs:
PathtoPublish: '$(DiffFile)'
ArtifactName: 'format.diff'
- task: PowerShell@2
displayName: '*** Test Modified Ports for ${{ parameters.triplet }}'
displayName: '*** Test Modified Ports'
inputs:
failOnStderr: true
filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
Expand All @@ -69,43 +81,7 @@ jobs:
displayName: 'Validate version files'
condition: eq('${{ parameters.triplet }}', '${{ variables.ExtraChecksTriplet }}')
inputs:
targetType: inline
script: |
./vcpkg.exe --feature-flags=versions x-ci-verify-versions --verbose |
ForEach-Object -Begin {
$long_error = ''
} -Process {
if ($long_error -ne '' -and $_ -match '^$|^ ') {
# Extend multi-line message
$long_error = -join($long_error, "%0D%0A", $_ -replace '^ ','' `
-replace '(git add) [^ ]*\\ports\\([^ ]*)', '$1 ports/$2' )
} else {
if ($long_error -ne '') {
# Flush multi-line message
$long_error
$long_error = ''
}
if ($_ -match '^Error: ') {
# Start multi-line message
$long_error = $_ -replace '^Error: ', '##vso[task.logissue type=error]' `
-replace '(^##vso[^\]]*)](.*) [^ ]*\\versions\\(.-)\\(.*.json)(.*)', '$1;sourcepath=versions/$3/$4;linenumber=2]$2 version/$3/$4$5'
} else {
# Normal line
$_
}
}
} -End {
if ($long_error -ne '') {
# Flush multi-line message
$long_error
}
}
pwsh: true
- task: PowerShell@2
displayName: 'Report on Disk Space After Build'
condition: always()
inputs:
filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'
filePath: 'scripts/azure-pipelines/windows/validate-version-files.ps1'
pwsh: true
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: failure logs for ${{ parameters.triplet }}'
Expand Down
7 changes: 7 additions & 0 deletions scripts/azure-pipelines/windows/bootstrap-from-source.cmd
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=x86 -host_arch=x86
git clone https://github.com/microsoft/vcpkg-tool vcpkg-tool
git -C vcpkg-tool switch -d %1
rmdir /s /q build.x86.release > nul 2> nul
cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=OFF -DVCPKG_WARNINGS_AS_ERRORS=OFF -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=OFF -B build.x86.release -S vcpkg-tool
ninja.exe -C build.x86.release
move build.x86.release\vcpkg.exe vcpkg.exe
29 changes: 29 additions & 0 deletions scripts/azure-pipelines/windows/validate-version-files.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
./vcpkg.exe --feature-flags=versions x-ci-verify-versions --verbose |
ForEach-Object -Begin {
$long_error = ''
} -Process {
if ($long_error -ne '' -and $_ -match '^$|^ ') {
# Extend multi-line message
$long_error = -join($long_error, "%0D%0A", $_ -replace '^ ','' `
-replace '(git add) [^ ]*\\ports\\([^ ]*)', '$1 ports/$2' )
} else {
if ($long_error -ne '') {
# Flush multi-line message
$long_error
$long_error = ''
}
if ($_ -match '^Error: ') {
# Start multi-line message
$long_error = $_ -replace '^Error: ', '##vso[task.logissue type=error]' `
-replace '(^##vso[^\]]*)](.*) [^ ]*\\versions\\(.-)\\(.*.json)(.*)', '$1;sourcepath=versions/$3/$4;linenumber=2]$2 version/$3/$4$5'
} else {
# Normal line
$_
}
}
} -End {
if ($long_error -ne '') {
# Flush multi-line message
$long_error
}
}

0 comments on commit 4c82b77

Please sign in to comment.