Move AZP Setup to Docs (microsoft#1665)

CyberFlameGO · Jun 1, 2021 · 019d5f1 · 019d5f1
1 parent 30593c3
commit 019d5f1
Showing 1 changed file with 263 additions and 0 deletions.
diff --git a/docs/AZP.md b/docs/AZP.md
@@ -0,0 +1,263 @@
+# Azure Pipelines Machine Setup
+
+The following are a set of instructions to set up an Azure Pipeline test machine for running the latest Windows builds on private/custom hardware. Both VMs and HW are supported/allowed.
+
+## Windows OS Install
+
+- Install the latest Windows Server Datacenter (desktop) SKU, ideally from the RS_PRERELEASE branch if possible, since symbols persist the longest for that branch.
+- Log in as Administrator.
+- Make sure to activate it. If necessary grab a product key from the internal website.
+
+## Machine Setup
+
+- Copy `sfpcopy.exe` into C:\Windows\System32
+- Disable Windows Defender
+- Install the latest PowerShell: `iex "& { $(irm https://aka.ms/install-powershell.ps1) } -UseMSI"`
+- Download and install VS build tools:
+  - https://visualstudio.microsoft.com/thank-you-downloading-visual-studio/?sku=BuildTools&rel=16
+  - Specifically make sure **C++ Build Tools** is installed
+- Enable the kernel debugger:
+```
+bcdedit /dbgsettings NET HOSTIP:192.168.1.160 PORT:50000 /noumex
+bcdedit -deletevalue {dbgsettings} hostip
+bcdedit /debug on
+```
+- Start windbg on debug server machine to connect with autogenerated key from above.
+- **Reboot**
+
+## Agent Setup
+
+- Download the agent: https://vstsagentpackage.azureedge.net/agent/2.172.2/vsts-agent-win-x64-2.172.2.zip
+- Unzip the contents to `C:\a`
+- Run `config.cmd`
+  - Connect to `https://dev.azure.com/ms`
+  - Use the PAT (ping me to ask for it)
+  - Add to `MsQuic-Win-Latest` pool
+  - Use `w` for the work folder name
+  - Run it as a service, using the default account
+- Open 'Services' and find the Azure Pipelines service that was created
+- Change the service to run a Local System
+  - Then stop and restart it
+- Change the service to run as Administrator
+  - Then stop and restart it
+
+## Script for preparing VMs for use in the AZP pool
+```
+param (
+    [Parameter(Mandatory = $true)]
+    [ValidateSet("prepareseed", "create", "config", "delete", "launchdebug")]
+    [string]$Mode,
+
+    [Parameter(Mandatory = $false)]
+    [string]$SeedVhdLocation = 'c:\SeedVHDs',
+
+    [Parameter(Mandatory = $false)]
+    [string]$SeedVhd = 'seed.vhdx',
+
+    [Parameter(Mandatory = $false)]
+    [string]$VhdLocation = 'c:\users\public\documents\Hyper-V\Virtual hard disks',
+
+    [Parameter(Mandatory = $false)]
+    [long]$VhdSize = 40GB,
+
+    [Parameter(Mandatory = $false)]
+    [string]$UnattendLocation = "$($PSScriptRoot)",
+
+    [Parameter(Mandatory = $false)]
+    [string]$PAT = '',
+
+    [Parameter(Mandatory = $false)]
+    [long]$VmCores = 4,
+
+    [Parameter(Mandatory = $false)]
+    [long]$MaximumVmMemory = 4096,
+
+    [Parameter(Mandatory = $false)]
+    [long]$VmsToProcess = 14,
+
+    [Parameter(Mandatory = $false)]
+    [long]$VmNumberStart = 1,
+
+    [Parameter(Mandatory = $false)]
+    [string]$VmNamePrefix = "AZP",
+
+    [Parameter(Mandatory = $false)]
+    [string]$VmPassword = "Very!SecurePassword2",
+
+    [Parameter(Mandatory = $false)]
+    [string]$AgentPool = 'MsQuic-Win-Latest',
+
+    [Parameter(Mandatory = $false)]
+    [string]$AgentUrl = 'https://dev.azure.com/ms',
+
+    [Parameter(Mandatory = $false)]
+    [long]$DebugPortBase = 50000
+)
+
+$ProgressPreference = 'SilentlyContinue'
+
+$AzpDebugKey = "something.very.very.secret"
+
+function ExecuteScriptWithMountedVhd($vhd, $scriptblock)
+{
+    $vhdname = (Split-Path $vhd -Leaf).substring(0,8)
+    $vhdmountdir = "$($Env:Temp)\vhdmount\$vhdname"
+    if (Test-Path $vhdmountdir) {
+        throw "VHD mount path already exists."
+    }
+    mkdir $vhdmountdir > $null
+    dism.exe /mount-image /ImageFile:$vhd /index:1 /mountdir:$vhdmountdir
+    & $scriptblock
+    dism.exe /unmount-image /mountdir:$vhdmountdir /commit
+    Remove-Item -Path $vhdmountdir -Force > $null
+}
+
+function AddUnattendFile($vhd, $unattendfile)
+{
+    ExecuteScriptWithMountedVhd $vhd {
+        dism.exe /image:$vhdmountdir /apply-unattend:$unattendfile
+        Copy-Item $unattendfile $vhdmountdir
+    }
+}
+
+function CreateVM($VmName, $VmNumber)
+{
+    $VhdExtension = $SeedVhd.split('.')[-1]
+    $VhdPath = "$(Join-Path $VhdLocation $VmName).$($VhdExtension)"
+
+    if (-not (Test-Path $VhdPath)) {
+        New-VHD -Path $VhdPath -ParentPath (Join-Path $SeedVhdLocation $SeedVhd) -Differencing
+    }
+
+    $VMGeneration = 1
+    if ($VhdExtension.Contains("vhdx")) {
+        $VmGeneration = 2
+    }
+
+    New-Vm -Name $VmName -VHDPath $VhdPath -Generation $VMGeneration -SwitchName "ExternalSwitch"
+
+    Set-VMProcessor -VMName $VmName -Count $VmCores
+
+    Set-VMMemory -VMName $VmName -DynamicMemoryEnabled $true -MinimumBytes (512 * 1024 * 1024) -StartupBytes (1024 *1024 * 1024) -MaximumBytes ($MaximumVmMemory * 1024 * 1024)
+
+    if ($VMGeneration -eq 2) {
+        Set-VMFirmware -VMName $VmName -FirstBootDevice (Get-VMHardDiskDrive -VMName $VmName) -EnableSecureBoot Off
+    }
+
+    Start-VM $VmName
+
+    # Wait for VM to start - never gets to "Healthy"
+    # while ((Get-VM -Name $VmName).HeartBeat -ne  'OkApplicationsHealthy') {
+    #     Write-Host "VM state is: $((Get-VM -Name $VmName).HeartBeat) Sleeping 5 seconds..."
+    #     Start-Sleep -Seconds 5
+    # }
+}
+
+function ConfigureVM($VmName, $VmNumber)
+{
+    $block = {
+        param($Port, $HostIP, $VmName, $Token, $DebugKey, $Password, $PoolName, $URL)
+        Set-MpPreference -DisableRealtimeMonitoring $true
+        Copy-Item c:\sfpcopy.exe c:\Windows\System32
+        Invoke-Expression "& { $(Invoke-RestMethod https://aka.ms/install-powershell.ps1) } -UseMSI"
+        bcdedit.exe /dbgsettings NET HOSTIPV6:$HostIP PORT:$Port KEY:$DebugKey /noumex
+        bcdedit.exe /debug on
+        # Install VS build tools
+        c:\vs_buildtools.exe --add Microsoft.VisualStudio.Workload.MSBuildTools --add Microsoft.VisualStudio.Workload.VCTools --IncludeRecommended -q --force --wait
+        # Add agent to pool
+        c:\a\config.cmd --unattended --URL "$($URL)" --auth pat --token "$($Token)" --pool "$($PoolName)" --agent "$($VmName)" --replace --work "c:\a\w" --runAsService
+        # Enable services to run as Administrator
+        secedit.exe /export /cfg "$($Env:Temp)\secconfig.cfg"
+        (Get-Content "$($Env:Temp)\secconfig.cfg") -replace 'SeServiceLogonRight = (?<SIDs>.*)', 'SeServiceLogonRight = Administrator,$1' | Set-Content "$($Env:Temp)\secconfig.cfg"
+        secedit.exe /configure /db c:\windows\security\local.sdb /cfg "$($Env:Temp)\secconfig.cfg" /areas USER_RIGHTS
+        # Set AZP service to run as Administrator
+        #$cred = New-Object System.Management.Automation.PSCredential ('Administrator', (ConvertTo-SecureString $Password -AsPlainText -Force))
+        #& "C:\Program Files\Powershell\7\pwsh.exe" -Command Set-Service -Name "vsts*" -Credential (New-Object System.Management.Automation.PSCredential ('Administrator', (ConvertTo-SecureString $Password -AsPlainText -Force)))
+        #Get-Service vsts* | Restart-Service
+        $ServiceName = (get-service vsts*).name
+        $WmiObject = Get-WMIObject Win32_Service -filter "name='$ServiceName'"
+        $StopStatus = $WmiObject.StopService()
+        If ($StopStatus.ReturnValue -eq "0") {
+            Write-host "The service '$ServiceName' Stopped successfully"
+        }
+        $ChangeStatus = $WmiObject.change($null,$null,$null,$null,$null,$null,'.\Administrator',$Password,$null,$null,$null)
+        If ($ChangeStatus.ReturnValue -eq "0") {
+            Write-host "Set User Name sucessfully for the service '$ServiceName'"
+        }
+        Start-Sleep 5
+        $StartStatus = $WmiObject.StartService()
+        If ($StartStatus.ReturnValue -eq "0") {
+            Write-host "The service '$ServiceName' Started successfully"
+        }
+        # Visual studio installer doesn't respect the --wait parameter, so we can't automatically reboot
+    }
+
+    $DebugPort = $DebugPortBase + $VmNumber
+    $HostIP =  (get-netipaddress -AddressFamily IPv6 -AddressState Preferred -InterfaceAlias "vEthernet (ExternalSwitch)" -ValidLifetime ([TimeSpan]::MaxValue)).IPAddress.Split('%')[0]
+
+    $password = ConvertTo-SecureString $VmPassword -AsPlainText -Force
+    $credential = New-Object System.Management.Automation.PSCredential ('Administrator', $password)
+
+    #start up WinDbgX
+    windbgx.exe -y c:\symbols -k net:port=$DebugPort,key=$AzpDebugKey
+
+    #Configure VM
+    Invoke-Command -VmName $VmName -Credential $credential -ScriptBlock $block -ArgumentList $DebugPort,$HostIP,$VmName,$PAT, $AzpDebugKey,$VmPassword,$AgentPool,$AgentUrl
+
+    Write-Host "Finished setting up $VmName"
+}
+
+function DeleteVM($VmName)
+{
+    if ((Get-VM -Name $VmName).State -eq 'Running') {
+
+        $block = {
+            param($Token)
+            C:\a\config.cmd remove --unattended --auth pat --token "$($Token)"
+        }
+
+        $password = ConvertTo-SecureString $VmPassword -AsPlainText -Force
+        $credential = New-Object System.Management.Automation.PSCredential ('Administrator', $password)
+
+        Invoke-Command -VmName $VmName -Credential $credential -ScriptBlock $block -ArgumentList $PAT
+
+        # Turn off running VM
+        Stop-VM -Name $VmName -TurnOff -Force
+    }
+   # Remove VHDs (and snapshot VHDs) before the VM
+   Remove-Item -Path "$(Join-Path $VhdLocation $VmName)*" -Force
+   Remove-VM -Name $VmName -Force
+}
+
+#
+# Actually run the script
+#
+
+if ($Mode -eq "prepareseed") {
+    $Vhd = Join-Path $SeedVhdLocation $SeedVhd
+    if ((Get-VHD $Vhd).Size -lt $VhdSize) {
+        Resize-VHD -Path $Vhd -SizeBytes $VhdSize
+    }
+    AddUnattendFile $Vhd "$($UnattendLocation)\unattend.xml"
+    ExecuteScriptWithMountedVhd $Vhd {
+        Invoke-WebRequest -Uri "https://vstsagentpackage.azureedge.net/agent/2.172.2/vsts-agent-win-x64-2.172.2.zip" -OutFile "$($vhdmountdir)\vsts-agent-win-x64-2.172.2.zip"
+        Expand-Archive -Path "$($vhdmountdir)\vsts-agent-win-x64-2.172.2.zip" -DestinationPath "$($vhdmountdir)\a" -Force
+        Copy-Item "vs_buildtools.exe" "$($vhdmountdir)\vs_buildtools.exe"
+        Copy-Item "sfpcopy.exe" "$($vhdmountdir)\sfpcopy.exe"
+    }
+} else {
+    for ($i = $VmNumberStart; $i -le $VmsToProcess; $i++) {
+        if ($Mode -eq "create") {
+            CreateVM "$($VmNamePrefix)-$($i)" $i
+        } elseif ($Mode -eq "config") {
+            ConfigureVM "$($VmNamePrefix)-$($i)" $i
+        } elseif ($Mode -eq "delete") {
+             $VmName = "$($VmNamePrefix)-$($i)"
+             DeleteVm $VmName
+        } elseif ($Mode -eq "launchdebug") {
+            $DebugPort = $DebugPortBase + $i
+            windbgx.exe -y c:\symbols -k net:port=$DebugPort,key=$AzpDebugKey
+        }
+    }
+}
+```