[vcpkg] Android CI support (microsoft#29406)

* initial

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

scripts/azure-pipelines/android/Dockerfile

@@ -0,0 +1,55 @@
+FROM ubuntu:20.04
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN \
+  apt-get update && \
+  apt-get -y upgrade && \
+  apt-get -y --no-install-recommends install git g++ wget curl zip vim \
+  pkg-config tar cmake unzip ca-certificates
+
+# Download Android NDK
+RUN \
+  wget https://dl.google.com/android/repository/android-ndk-r25c-linux.zip && \
+  unzip android-ndk-r25c-linux.zip && \
+  rm -rf android-ndk-r25c-linux.zip
+
+ENV ANDROID_NDK_HOME /android-ndk-r25c
+
+# Download Powershell
+RUN \
+  wget -q https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb && \
+  dpkg -i packages-microsoft-prod.deb && \
+  rm -f packages-microsoft-prod.deb && \
+  apt-get -y install software-properties-common && \
+  apt-get update && \
+  add-apt-repository universe && \
+  apt-get -y install powershell
+
+RUN \
+  apt-get -y --no-install-recommends install autoconf nasm bison python2 \
+  flex build-essential libtool libtool-bin gettext automake autoconf-archive
+
+# python related
+RUN \
+  apt-get -y --no-install-recommends install python3-setuptools python3-pip python3-venv
+
+# freeglut
+RUN \
+  apt-get -y --no-install-recommends install libxi-dev libgl1-mesa-dev \
+  libglu1-mesa-dev mesa-common-dev libxrandr-dev libxxf86vm-dev
+
+# glfw3
+RUN \
+  apt-get -y --no-install-recommends install libxinerama-dev libxcursor-dev
+
+# required by qt5-base
+RUN \
+  apt-get -y --no-install-recommends install libxext-dev libxfixes-dev libxrender-dev \
+  libxcb1-dev libx11-xcb-dev libxcb-glx0-dev libxcb-util0-dev \
+  libxkbcommon-dev libxcb-keysyms1-dev \
+  libxcb-image0-dev libxcb-shm0-dev libxcb-icccm4-dev libxcb-sync-dev \
+  libxcb-xfixes0-dev libxcb-shape0-dev libxcb-randr0-dev \
+  libxcb-render-util0-dev libxcb-xinerama0-dev libxcb-xkb-dev libxcb-xinput-dev \
+  libxcb-cursor-dev libxkbcommon-x11-dev
+
+WORKDIR /vcpkg

scripts/azure-pipelines/android/azure-pipelines.yml

@@ -0,0 +1,136 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+#
+
+parameters:
+  - name: vcpkgToolSha
+    displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap'
+    type: string
+    default: 'use default'
+  - name: triplet
+    type: string
+  - name: jobName
+    type: string
+  - name: poolName
+    type: string
+  - name: dockerImage
+    type: string
+  - name: tripletPattern
+    displayName: 'Enable the triplets which contain this substring'
+    type: string
+    default: ''
+
+jobs:
+- job: ${{ parameters.jobName }}
+  condition: and(succeeded(), contains('^${{ parameters.triplet }}$', '${{ parameters.tripletPattern }}'))
+  pool:
+    name: ${{ parameters.poolName }}
+  workspace:
+    clean: resources
+  timeoutInMinutes: 1440 # 1 day
+  cancelTimeoutInMinutes: 1
+  variables:
+  - name: WORKING_ROOT
+    value: /mnt/vcpkg-ci
+  - name: VCPKG_DOWNLOADS
+    value: /mnt/vcpkg-ci/downloads
+  - group: vcpkg-asset-caching-credentials
+  - name: X_VCPKG_ASSET_SOURCES
+    value: "x-azurl,$(root-url-ea),$(sas-ea),readwrite"
+  - group: vcpkg-binary-caching-credentials
+  - name: X_VCPKG_BINARY_SOURCE_STUB
+    value: "x-azblob,$(root-bin-url-ea),$(sas-bin-ea)"
+
+  steps:
+    # Note: /mnt is the Azure machines' temporary disk.
+  - bash: |
+      curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+      az login --identity
+      az acr login --name AndContainerRegistry
+    displayName: 'Set up managed identity'
+  - bash: |
+      sudo mkdir /home/agent -m=777
+      sudo chown `id -u` /home/agent
+      sudo mkdir ${{ variables.WORKING_ROOT }} -m=777
+      sudo rm -rf ${{ variables.WORKING_ROOT }}/failure-logs
+      sudo mkdir ${{ variables.WORKING_ROOT }}/failure-logs -m=777
+      sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} -m=777
+      exit 0
+    displayName: 'Create working directories'
+  - bash: |
+      docker run --rm \
+      --mount type=bind,source=$(Build.Repository.LocalPath),target=/vcpkg \
+      ${{ parameters.dockerImage }} \
+      ./bootstrap-vcpkg.sh
+    displayName: 'Bootstrap vcpkg'
+    condition: eq('use default', '${{ parameters.vcpkgToolSha }}')
+  - bash: |
+      docker run --rm \
+      --mount type=bind,source=$(Build.Repository.LocalPath),target=/vcpkg \
+      ${{ parameters.dockerImage }} \
+      ./scripts/azure-pipelines/bootstrap-from-source.sh ${{ parameters.vcpkgToolSha }}
+    displayName: "Build vcpkg with CMake"
+    condition: ne('use default', '${{ parameters.vcpkgToolSha }}')
+  - bash: |
+      CONTAINERS=$(docker ps -a -q)
+      if [ -n "$CONTAINERS" ]; then
+        docker rm -f $CONTAINERS
+      fi
+    displayName: 'Stop and remove all docker containers'
+  - bash: |
+      docker pull ${{ parameters.dockerImage }}
+    displayName: 'Pull Docker Container'
+  - task: Bash@3
+    inputs:
+      targetType: inline
+      script: |
+        USER=$(id --user)
+        docker run --init -i \
+        -a stderr \
+        -a stdout \
+        --user $USER \
+        --mount type=bind,source=$(Build.Repository.LocalPath),target=/vcpkg \
+        --mount type=bind,source=$(WORKING_ROOT)/failure-logs,target=/vcpkg/failure-logs \
+        --mount type=bind,source=/mnt/vcpkg-ci,target=/mnt/vcpkg-ci \
+        --env X_VCPKG_ASSET_SOURCES="x-azurl,$(root-url-ea),$(sas-ea),readwrite" \
+        ${{ parameters.dockerImage }} \
+        pwsh \
+         -File /vcpkg/scripts/azure-pipelines/test-modified-ports.ps1 \
+         -Triplet ${{ parameters.triplet }} \
+         -BuildReason $(Build.Reason) \
+         -BinarySourceStub "$(X_VCPKG_BINARY_SOURCE_STUB)" \
+         -WorkingRoot ${{ variables.WORKING_ROOT }} \
+         -IsLinuxHost
+      failOnStderr: true
+    displayName: '*** Test Modified Ports'
+  - bash: |
+      CONTAINERS=$(docker ps -a -q)
+      if [ -n "$CONTAINERS" ]; then
+        docker rm -f $CONTAINERS
+      fi
+    displayName: 'Stop and remove all docker containers'
+    condition: always()
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: failure logs for ${{ parameters.triplet }}'
+    inputs:
+      PathtoPublish: '$(WORKING_ROOT)/failure-logs'
+      ArtifactName: 'failure logs for ${{ parameters.triplet }}'
+    condition: ne(variables['FAILURE_LOGS_EMPTY'], 'True')
+  - bash: |
+      python3 scripts/file_script.py /mnt/vcpkg-ci/installed/vcpkg/info/
+    displayName: 'Build a file list for all packages'
+    condition: always()
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: file lists for ${{ parameters.triplet }}'
+    condition: always()
+    inputs:
+      PathtoPublish: scripts/list_files
+      ArtifactName: 'file lists for ${{ parameters.triplet }}'
+  - task: PublishTestResults@2
+    displayName: 'Publish Test Results'
+    condition: ne(variables['XML_RESULTS_FILE'], '')
+    inputs:
+      testRunTitle: ${{ parameters.triplet }}
+      testResultsFormat: xUnit
+      testResultsFiles: $(XML_RESULTS_FILE)
+      platform: ${{ parameters.triplet }}

scripts/azure-pipelines/android/create-docker-image.ps1

@@ -0,0 +1,32 @@
+# Create Docker image for Android
+
+$Location = 'eastasia'
+$Date = (Get-Date -Format 'yyyy-MM-dd')
+$ResourceGroupName = "And-Registry"
+$ContainerRegistryName = "AndContainerRegistry"
+$ErrorActionPreference = 'Stop'
+
+Get-AzResourceGroup -Name $ResourceGroupName -ErrorVariable error -ErrorAction SilentlyContinue
+if ($error) {
+    New-AzResourceGroup -Name $ResourceGroupName -Location $Location
+    New-AzContainerRegistry -ResourceGroupName $ResourceGroupName -Name $ContainerRegistryName -EnableAdminUser -Sku Basic
+}
+
+$registry = Get-AzContainerRegistry -ResourceGroupName $ResourceGroupName -Name $ContainerRegistryName
+Connect-AzContainerRegistry -Name $registry.Name
+
+$imageName = "vcpkg-android"
+docker build . -t $imageName
+
+$remote = [string]::Format('andcontainerregistry.azurecr.io/{0}:{1}', $imageName, $Date)
+docker tag $imageName $remote
+
+docker push $remote
+
+#removes from local environment
+docker rmi --force $remote $imageName
+
+# pulls and runs ...
+docker logout
+
+Write-Host "Remote: $remote"

scripts/azure-pipelines/android/create-image.ps1

@@ -0,0 +1 @@
+../linux/create-image.ps1 -Prefix "And-" -ProvisionScript (Get-ChildItem .\provision-image.sh).fullname

scripts/azure-pipelines/android/create-vmss.ps1

@@ -0,0 +1,6 @@
+[CmdLetBinding()]
+Param(
+  [parameter(Mandatory=$true)]
+  [string]$ImageName
+)
+../linux/create-vmss.ps1 -ImageName $ImageName -Prefix "PrAnd-"

scripts/azure-pipelines/android/provision-image.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+#
+# This script is to set up the machine for the Docker host.
+
+export DEBIAN_FRONTEND=noninteractive
+
+## Docker
+apt-get -y --no-install-recommends install ca-certificates gnupg lsb-release
+mkdir -p /etc/apt/keyrings
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
+  $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
+apt-get update
+apt-get -y --no-install-recommends install docker-ce docker-ce-cli

scripts/azure-pipelines/android/setup-managed-identity.ps1

@@ -0,0 +1,17 @@
+# Create a system managed identity
+# Adding to existing Scale set
+$scaleSetResourceGroup = "PrAnd-2023-04-17"
+$scaleSetName = "PrAnd-2023-04-17Vmss"
+
+$vm = Get-AzVMss -ResourceGroupName $scaleSetResourceGroup -Name $scaleSetName
+Update-AzVMss -ResourceGroupName $scaleSetResourceGroup -VMScaleSetName $scaleSetName -IdentityType SystemAssigned
+
+$spID = $vm.Identity.PrincipalId
+
+$acrGroup = "And-Registry"
+$acrName = "AndContainerRegistry"
+
+$resourceID = (Get-AzContainerRegistry -ResourceGroupName $acrGroup -Name $acrName).Id
+
+# needs admin privileges
+New-AzRoleAssignment -ObjectId $spID -Scope $resourceID -RoleDefinitionName AcrPull

scripts/azure-pipelines/azure-pipelines.yml

@@ -5,6 +5,8 @@ variables:
   windows-pool: 'PrWin-2023-04-12'
   linux-pool: 'PrLin-2023-04-12'
   osx-pool: 'PrOsx-2022-02-04'
+  android-pool: 'PrAnd-2023-04-17'
+  linux-docker-image: 'andcontainerregistry.azurecr.io/vcpkg-android:2023-04-20'
 
 parameters:
   - name: vcpkgToolSha
@@ -84,3 +86,30 @@ jobs:
     poolName: $(linux-pool)
     vcpkgToolSha: ${{ parameters.vcpkgToolSha }}
     tripletPattern: ${{ parameters.tripletPattern }}
+
+- template: android/azure-pipelines.yml
+  parameters:
+    triplet: arm-neon-android
+    jobName: arm_neon_android
+    poolName: $(android-pool)
+    dockerImage: $(linux-docker-image)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}
+    tripletPattern: ${{ parameters.tripletPattern }}
+
+- template: android/azure-pipelines.yml
+  parameters:
+    triplet: x64-android
+    jobName: x64_android
+    poolName: $(android-pool)
+    dockerImage: $(linux-docker-image)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}
+    tripletPattern: ${{ parameters.tripletPattern }}
+
+- template: android/azure-pipelines.yml
+  parameters:
+    triplet: arm64-android
+    jobName: arm64_android
+    poolName: $(android-pool)
+    dockerImage: $(linux-docker-image)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}
+    tripletPattern: ${{ parameters.tripletPattern }}

scripts/azure-pipelines/linux/azure-pipelines.yml

@@ -54,7 +54,7 @@ jobs:
     inputs:
       failOnStderr: true
       filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
-      arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -BinarySourceStub "$(X_VCPKG_BINARY_SOURCE_STUB)" -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'
+      arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -BinarySourceStub "$(X_VCPKG_BINARY_SOURCE_STUB)" -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) -IsLinuxHost'
       pwsh: true
   - task: PublishBuildArtifacts@1
     displayName: 'Publish Artifact: failure logs for x64-linux'

scripts/azure-pipelines/linux/create-image.ps1

@@ -15,8 +15,15 @@ or are running from Azure Cloud Shell.
 This script assumes you have installed the OpenSSH Client optional Windows component.
 #>
 
+[CmdLetBinding()]
+Param(
+  [parameter(Mandatory=$false)]
+  [string]$Prefix = "Lin-",
+  [parameter(Mandatory=$false)]
+  [string]$ProvisionScript
+)
+
 $Location = 'eastasia'
-$Prefix = 'Lin-'
 $Prefix += (Get-Date -Format 'yyyy-MM-dd')
 $VMSize = 'Standard_D8a_v4'
 $ProtoVMName = 'PROTOTYPE'
@@ -126,11 +133,14 @@ Write-Progress `
   -Status 'Running provisioning script provision-image.sh in VM' `
   -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
 
+if($ProvisionScript.Length -eq 0) {
+  $ProvisionScript = "$PSScriptRoot\provision-image.sh"
+}
 $ProvisionImageResult = Invoke-AzVMRunCommandWithRetries `
   -ResourceGroupName $ResourceGroupName `
   -VMName $ProtoVMName `
   -CommandId 'RunShellScript' `
-  -ScriptPath "$PSScriptRoot\provision-image.sh"
+  -ScriptPath $ProvisionScript
 
 Write-Host "provision-image.sh output: $($ProvisionImageResult.value.Message)"
 

scripts/azure-pipelines/linux/create-vmss.ps1

@@ -25,11 +25,12 @@ The name of the image to deploy into the scale set.
 [CmdLetBinding()]
 Param(
   [parameter(Mandatory=$true)]
-  [string]$ImageName
+  [string]$ImageName,
+  [parameter(Mandatory=$false)]
+  [string]$Prefix = "PrLin-"
 )
 
 $Location = 'eastasia'
-$Prefix = 'PrLin-'
 $Prefix += (Get-Date -Format 'yyyy-MM-dd')
 $VMSize = 'Standard_D32a_v4'
 $LiveVMPrefix = 'BUILD'

scripts/azure-pipelines/test-modified-ports.ps1

@@ -53,7 +53,8 @@ Param(
     $BinarySourceStub = $null,
     [String]$BuildReason = $null,
     [switch]$NoParentHashes = $false,
-    [switch]$PassingIsPassing = $false
+    [switch]$PassingIsPassing = $false,
+    [switch]$IsLinuxHost = $false
 )
 
 if (-Not ((Test-Path "triplets/$Triplet.cmake") -or (Test-Path "triplets/community/$Triplet.cmake"))) {
@@ -104,7 +105,7 @@ if ([string]::IsNullOrWhiteSpace($BinarySourceStub)) {
     $cachingArgs += @("--binarysource=clear;$BinarySourceStub,$binaryCachingMode")
 }
 
-if ($Triplet -eq 'x64-linux') {
+if ($IsLinuxHost) {
     $env:HOME = '/home/agent'
     $executableExtension = [string]::Empty
 }