Publish the official build to blob storage (dotnet#1092)

* Publish installers and checksums

- Use the arcade publishing process to publish the installers and checksums
- Remove old blob publishing
- Calculate the product version using an always-suffixed version
- Change the blob path that we should upload to to be Runtime/<version> instead of assets/core-setup/.. to line up with the desired paths.

* Updates for dotnet/runtime

Use InstallerTasksAssemblyPath.

Fail fast on RuntimeNupkgFile item duplicates.

Find some more symbol packages. AllConfigurations symbol nupkgs weren't detected due to different artifact arrangement. There may be more missing, however I did this to do unblock some quick local validation so I'm including it.

Co-authored-by: Matt Mitchell <[email protected]>

eng/pipelines/official/stages/publish.yml

@@ -33,6 +33,8 @@ stages:
     # Allow symbol publish to emit expected warnings without failing the build. Include single
     # quotes inside the string so that it passes through to MSBuild without script interference.
     symbolPublishingAdditionalParameters: "'-warnAsError:$false'"
+    # Publish to blob storage.
+    publishInstallersAndChecksums: true
     # Enable SDL validation, passing through values from the 'core-setup-sdl-validation' group.
     SDLValidationParameters:
       enable: false # TODO: (Consolidation) Decide who owns SDL validation errors and enable. https://github.com/dotnet/runtime/issues/1027

src/installer/Directory.Build.targets

@@ -90,7 +90,6 @@
       <HostPolicyVersion Condition="'$(UseShippedHostPolicyPackage)' != 'true'">$(ProductVersion)</HostPolicyVersion>
       <HostPolicyVersion Condition="'$(UseShippedHostPolicyPackage)' == 'true'">2.0.0</HostPolicyVersion>
 
-      <BinariesRelativePath>Runtime/$(SharedFrameworkNugetVersion)/</BinariesRelativePath>
       <InstallersRelativePath>Runtime/$(SharedFrameworkNugetVersion)/</InstallersRelativePath>
     </PropertyGroup>
 

src/installer/publish/Directory.Build.targets

@@ -33,14 +33,30 @@
         are published from every job. RID-agnostic nupkgs are built with the same ID/version by
         every job, so one specific job's outputs must be picked to sign and publish.
       -->
+
+      <!-- RID-specific framework packs. -->
       <RuntimeNupkgFile
         Include="
-          $(DownloadDirectory)**\runtime.*.nupkg;
-          $(DownloadDirectory)**\*.Runtime.*.nupkg;
-          $(DownloadDirectory)**\*.App.Host.*.nupkg;
-          $(DownloadDirectory)**\VS.Redist.Common.*.nupkg"
+          $(DownloadDirectory)**\Microsoft.*.Runtime.*.nupkg;
+          $(DownloadDirectory)**\Microsoft.*.App.Host.*.nupkg"
+        Exclude="@(DownloadedSymbolNupkgFile)" />
+
+      <!-- VS insertion packages, carrying RID-specific installers. -->
+      <RuntimeNupkgFile
+        Include="$(DownloadDirectory)**\VS.Redist.Common.*.nupkg"
         Exclude="@(DownloadedSymbolNupkgFile)" />
 
+      <!--
+        Runtime packages associated with some identity packages. Need to exclude "runtime.native.*"
+        because Libraries produces some "runtime.native.Foo" packages with
+        "runtime.<rid>.runtime.native.Foo" identity packages.
+      -->
+      <RuntimeNupkgFile
+        Include="$(DownloadDirectory)**\runtime.*.nupkg"
+        Exclude="
+          $(DownloadDirectory)**\runtime.native.*.nupkg;
+          @(DownloadedSymbolNupkgFile)" />
+
       <!--
         Packages that aren't matched above as RID-specific are considered RID-agnostic. Also include
         the AllConfigurations packages from the Libraries build.
@@ -61,9 +77,15 @@
         reasons, such as the VS insertion packages that transport MSIs. Symbol package validation
         will check for symbol completeness with file-by-file granularity rather than looking for
         missing symbols.nupkg files: https://github.com/dotnet/arcade/issues/2499.
+
+        Handles several conventions:
+        * NonShipping packages have symbol nupkgs that are Shipping.
+        * Shipping packages have symbol packages in a "symbols" subdirectory.
       -->
       <PotentialSymbolNupkgToPublishFile
-        Include="@(NupkgToPublishFile->Replace('\NonShipping\', '\Shipping\')->Replace('.nupkg', '.symbols.nupkg'))" />
+        Include="
+          @(NupkgToPublishFile->Replace('\NonShipping\', '\Shipping\')->Replace('.nupkg', '.symbols.nupkg'));
+          @(NupkgToPublishFile->Replace('\Shipping\', '\Shipping\symbols\')->Replace('.nupkg', '.symbols.nupkg'))" />
 
       <SymbolNupkgToPublishFile
         Include="@(PotentialSymbolNupkgToPublishFile)"
@@ -75,6 +97,16 @@
     <Error
       Condition="'@(SymbolNupkgToPublishFile)' == ''"
       Text="No symbol packages found." />
+
+    <!--
+      Duplicate RuntimeNupkgFile items mean artifact upload will fail, but only after another hour
+      of signing. Detect this early. It's possible to automatically "fix" this with Distinct(),
+      however the patterns should be fairly specific: this is likely a build infra mistake that
+      should be corrected.
+    -->
+    <Error
+      Text="Duplicate RuntimeNupkgFile entries for: %(RuntimeNupkgFile.Identity)"
+      Condition="@(RuntimeNupkgFile->Count()) &gt; 1" />
   </Target>
 
   <Import Project="..\Directory.Build.targets" />

src/installer/publish/prepare-artifacts.proj

@@ -1,6 +1,24 @@
 <Project>
   <Import Project="Sdk.props" Sdk="Microsoft.NET.Sdk" />
 
+  <UsingTask TaskName="GenerateChecksums" AssemblyFile="$(InstallerTasksAssemblyPath)" />
+
+  <PropertyGroup>
+    <SuppressFinalPackageVersion>true</SuppressFinalPackageVersion>
+  </PropertyGroup>
+
+  <Target Name="CreateChecksums">
+    <ItemGroup>
+      <ArtifactsForGeneratingChecksums
+        Include="@(UploadToBlobStorageFile)"
+        DestinationPath="%(FullPath)$(ChecksumExtension)" />
+
+      <GeneratedChecksumFile Include="@(ArtifactsForGeneratingChecksums -> '%(DestinationPath)')" />
+    </ItemGroup>
+
+    <GenerateChecksums Items="@(ArtifactsForGeneratingChecksums)" />
+  </Target>
+
   <Target Name="SignPackages"
           Condition="'$(SkipSigning)' != 'true' and '$(SignType)' != 'public'">
     <Message Importance="High" Text="Signing final packages" />
@@ -13,12 +31,17 @@
   <Target Name="UploadPreparedArtifactsToPipeline"
           DependsOnTargets="
             FindDownloadedArtifacts;
-            SignPackages">
+            SignPackages;
+            CreateChecksums">
     <PropertyGroup>
       <PreparedFileUploadDir>$(ArtifactsObjDir)PreparedFileUpload\</PreparedFileUploadDir>
     </PropertyGroup>
 
-    <Copy SourceFiles="@(UploadToBlobStorageFile)" DestinationFolder="$(PreparedFileUploadDir)">
+    <ItemGroup>
+      <AllFilesToBlobStorage Include="@(UploadToBlobStorageFile);@(UploadToBlobStorageFile)" />
+    </ItemGroup>
+
+    <Copy SourceFiles="@(AllFilesToBlobStorage)" DestinationFolder="$(PreparedFileUploadDir)">
       <Output TaskParameter="CopiedFiles" ItemName="CopiedUploadToBlobStorageFile" />
     </Copy>
 
@@ -96,9 +119,13 @@
 
       <ItemsToPush
         Include="@(UploadToBlobStorageFile)"
-        Exclude="@(NupkgToPublishFile);@(SymbolNupkgToPublishFile)" />
-      <ItemsToPush>
-        <RelativeBlobPath>assets/core-setup/$(InstallersRelativePath)%(Filename)%(Extension)</RelativeBlobPath>
+        Exclude="@(NupkgToPublishFile);@(SymbolNupkgToPublishFile)">
+        <RelativeBlobPath>$(InstallersRelativePath)%(Filename)%(Extension)</RelativeBlobPath>
+      </ItemsToPush>
+
+      <ItemsToPush Include="@(GeneratedChecksumFile)">
+        <RelativeBlobPath>$(InstallersRelativePath)%(Filename)%(Extension)</RelativeBlobPath>
+        <Category>Checksum</Category>
       </ItemsToPush>
     </ItemGroup>