Merge pull request dotnet/core-setup#7373 from wli3/port-fixes

port changes in SDK after Microsoft.NET.HostModel.AppHost is created

Commit migrated from dotnet/core-setup@f5776f8

src/installer/managed/Microsoft.NET.HostModel/AppHost/FailedToDeleteApphostException.cs

@@ -0,0 +1,21 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+
+namespace Microsoft.NET.HostModel.AppHost
+{
+    /// <summary>
+    /// Failed to delete apphost when trying to delete incomplete appphost
+    /// </summary>
+    public class FailedToDeleteApphostException : AppHostUpdateException
+    {
+        public string ExceptionMessage { get; }
+        public FailedToDeleteApphostException(string exceptionMessage)
+        {
+            ExceptionMessage = exceptionMessage;
+        }
+    }
+}
+

src/installer/managed/Microsoft.NET.HostModel/AppHost/HostWriter.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System;
+using System.Diagnostics;
 using System.IO;
 using System.IO.MemoryMappedFiles;
 using System.Text;
@@ -43,45 +44,68 @@ public static void CreateAppHost(
 
             BinaryUtils.CopyFile(appHostSourceFilePath, appHostDestinationFilePath);
 
-            // Re-write the destination apphost with the proper contents.
-            bool appHostIsPEImage = false;
-            using (var memoryMappedFile = MemoryMappedFile.CreateFromFile(appHostDestinationFilePath))
+            try
             {
-                using (MemoryMappedViewAccessor accessor = memoryMappedFile.CreateViewAccessor())
+                // Re-write the destination apphost with the proper contents.
+                bool appHostIsPEImage = false;
+                using (var memoryMappedFile = MemoryMappedFile.CreateFromFile(appHostDestinationFilePath))
                 {
-                    BinaryUtils.SearchAndReplace(accessor, AppBinaryPathPlaceholderSearchValue, bytesToWrite);
+                    using (MemoryMappedViewAccessor accessor = memoryMappedFile.CreateViewAccessor())
+                    {
+                        BinaryUtils.SearchAndReplace(accessor, AppBinaryPathPlaceholderSearchValue, bytesToWrite);
 
-                    appHostIsPEImage = BinaryUtils.IsPEImage(accessor);
+                        appHostIsPEImage = BinaryUtils.IsPEImage(accessor);
 
-                    if (windowsGraphicalUserInterface)
-                    {
-                        if (!appHostIsPEImage)
+                        if (windowsGraphicalUserInterface)
                         {
-                            throw new AppHostNotPEFileException();
+                            if (!appHostIsPEImage)
+                            {
+                                throw new AppHostNotPEFileException();
+                            }
+
+                            BinaryUtils.SetWindowsGraphicalUserInterfaceBit(accessor);
                         }
+                    }
+                }
 
-                        BinaryUtils.SetWindowsGraphicalUserInterfaceBit(accessor);
+                if (assemblyToCopyResorcesFrom != null && appHostIsPEImage)
+                {
+                    if (ResourceUpdater.IsSupportedOS())
+                    {
+                        // Copy resources from managed dll to the apphost
+                        new ResourceUpdater(appHostDestinationFilePath)
+                            .AddResourcesFromPEImage(assemblyToCopyResorcesFrom)
+                            .Update();
+                    }
+                    else
+                    {
+                        throw new AppHostCustomizationUnsupportedOSException();
                     }
                 }
-            }
 
-            if (assemblyToCopyResorcesFrom != null && appHostIsPEImage)
+                // Memory-mapped write does not updating last write time
+                File.SetLastWriteTimeUtc(appHostDestinationFilePath, DateTime.UtcNow);
+            }
+            catch (Exception ex)
             {
-                if (ResourceUpdater.IsSupportedOS())
+                FailedToDeleteApphostException failedToDeleteApphostException = null;
+                // Delete the destination file so we don't leave an unmodified apphost
+                try
                 {
-                    // Copy resources from managed dll to the apphost
-                    new ResourceUpdater(appHostDestinationFilePath)
-                        .AddResourcesFromPEImage(assemblyToCopyResorcesFrom)
-                        .Update();
+                    File.Delete(appHostDestinationFilePath);
                 }
-                else 
+                catch (Exception failedToDeleteEx) when (failedToDeleteEx is IOException || failedToDeleteEx is UnauthorizedAccessException)
                 {
-                    throw new AppHostCustomizationUnsupportedOSException();
+                    failedToDeleteApphostException = new FailedToDeleteApphostException(failedToDeleteEx.Message);
                 }
-            }
 
-            // Memory-mapped write does not updating last write time
-            File.SetLastWriteTimeUtc(appHostDestinationFilePath, DateTime.UtcNow);
+                if (failedToDeleteApphostException != null)
+                {
+                    throw new AggregateException(ex, failedToDeleteApphostException);
+                }
+
+                throw;
+            }
         }
 
         /// <summary>

src/installer/managed/Microsoft.NET.HostModel/AppHost/ResourceUpdater.cs

@@ -21,7 +21,7 @@ private sealed class Kernel32
             // Native methods for updating resources
             //
 
-            [DllImport(nameof(Kernel32), SetLastError=true)]
+            [DllImport(nameof(Kernel32), CharSet = CharSet.Unicode, SetLastError=true)]
             public static extern SafeUpdateHandle BeginUpdateResource(string pFileName,
                                                                       [MarshalAs(UnmanagedType.Bool)]bool bDeleteExistingResources);
 
@@ -71,7 +71,7 @@ public enum LoadLibraryFlags : uint
                 LOAD_LIBRARY_AS_IMAGE_RESOURCE = 0x00000020
             }
 
-            [DllImport(nameof(Kernel32), SetLastError=true)]
+            [DllImport(nameof(Kernel32), CharSet = CharSet.Unicode, SetLastError=true)]
             public static extern IntPtr LoadLibraryEx(string lpFileName,
                                                       IntPtr hReservedNull,
                                                       LoadLibraryFlags dwFlags);