Allow execution of R2R code in singlefile app on windows. (dotnet#40104)

* Keep executable PE sections executable in LayoutILOnly.

* Install unwind handlers if present

* do not do RtlAddFunctionTable on x86

* Check for Cor header before checking for R2R header.

* Delete function table in image dtor

* Avoid PAGE_EXECUTE_READWRITE, we should not need writeable for R2R

* Do relocations in ConvertedImageLayout only if the file is in a bundle. (otherwise R2R stays disabled)

src/coreclr/src/utilcode/pedecoder.cpp

@@ -1770,20 +1770,29 @@ void PEDecoder::LayoutILOnly(void *base, BOOL allowFullPE) const
                            PAGE_READONLY, &oldProtection))
         ThrowLastError();
 
-    // Finally, apply proper protection to copied sections
-    section = sectionStart;
-    while (section < sectionEnd)
+    // Finally, apply proper protection to copied sections    
+    for (section = sectionStart; section < sectionEnd; section++)
     {
         // Add appropriate page protection.
-        if ((section->Characteristics & VAL32(IMAGE_SCN_MEM_WRITE)) == 0)
+#if defined(CROSSGEN_COMPILE) || defined(TARGET_UNIX)
+        if (section->Characteristics & IMAGE_SCN_MEM_WRITE)
+            continue;
+
+        DWORD newProtection = PAGE_READONLY;
+#else
+        DWORD newProtection = section->Characteristics & IMAGE_SCN_MEM_EXECUTE ?
+            PAGE_EXECUTE_READ :
+            section->Characteristics & IMAGE_SCN_MEM_WRITE ?
+                PAGE_READWRITE :
+                PAGE_READONLY;
+#endif
+
+        if (!ClrVirtualProtect((void*)((BYTE*)base + VAL32(section->VirtualAddress)),
+            VAL32(section->Misc.VirtualSize),
+            newProtection, &oldProtection))
         {
-            if (!ClrVirtualProtect((void *) ((BYTE *)base + VAL32(section->VirtualAddress)),
-                                   VAL32(section->Misc.VirtualSize),
-                                   PAGE_READONLY, &oldProtection))
-                ThrowLastError();
+            ThrowLastError();
         }
-
-        section++;
     }
 
     RETURN;

src/coreclr/src/vm/peimagelayout.cpp

@@ -39,15 +39,15 @@ PEImageLayout* PEImageLayout::LoadFromFlat(PEImageLayout* pflatimage)
     return new ConvertedImageLayout(pflatimage);
 }
 
-PEImageLayout* PEImageLayout::LoadConverted(PEImage* pOwner)
+PEImageLayout* PEImageLayout::LoadConverted(PEImage* pOwner, BOOL isInBundle)
 {
     STANDARD_VM_CONTRACT;
 
     PEImageLayoutHolder pFlat(new FlatImageLayout(pOwner));
     if (!pFlat->CheckFormat())
         ThrowHR(COR_E_BADIMAGEFORMAT);
 
-    return new ConvertedImageLayout(pFlat);
+    return new ConvertedImageLayout(pFlat, isInBundle);
 }
 
 PEImageLayout* PEImageLayout::Load(PEImage* pOwner, BOOL bNTSafeLoad, BOOL bThrowOnError)
@@ -59,7 +59,7 @@ PEImageLayout* PEImageLayout::Load(PEImage* pOwner, BOOL bNTSafeLoad, BOOL bThro
 #else
     if (pOwner->IsInBundle())
     {
-        return PEImageLayout::LoadConverted(pOwner);
+        return PEImageLayout::LoadConverted(pOwner, true);
     }
 
     PEImageLayoutHolder pAlloc(new LoadedImageLayout(pOwner,bNTSafeLoad,bThrowOnError));
@@ -386,7 +386,7 @@ RawImageLayout::RawImageLayout(const void *mapped, PEImage* pOwner, BOOL bTakeOw
     IfFailThrow(Init((void*)mapped,(bool)(bFixedUp!=FALSE)));
 }
 
-ConvertedImageLayout::ConvertedImageLayout(PEImageLayout* source)
+ConvertedImageLayout::ConvertedImageLayout(PEImageLayout* source, BOOL isInBundle)
 {
     CONTRACTL
     {
@@ -397,33 +397,95 @@ ConvertedImageLayout::ConvertedImageLayout(PEImageLayout* source)
     m_Layout=LAYOUT_LOADED;
     m_pOwner=source->m_pOwner;
     _ASSERTE(!source->IsMapped());
+    m_isInBundle = isInBundle;
+
+    m_pExceptionDir = NULL;
 
     if (!source->HasNTHeaders())
         EEFileLoadException::Throw(GetPath(), COR_E_BADIMAGEFORMAT);
     LOG((LF_LOADER, LL_INFO100, "PEImage: Opening manually mapped stream\n"));
 
+#if !defined(CROSSGEN_COMPILE) && !defined(TARGET_UNIX)
+    // on Windows we may want to enable execution if the image contains R2R sections
+    // so must ensure the mapping is compatible with that
+    m_FileMap.Assign(WszCreateFileMapping(INVALID_HANDLE_VALUE, NULL,
+        PAGE_EXECUTE_READWRITE, 0,
+        source->GetVirtualSize(), NULL));
 
+    DWORD allAccess = FILE_MAP_EXECUTE | FILE_MAP_WRITE;
+#else
     m_FileMap.Assign(WszCreateFileMapping(INVALID_HANDLE_VALUE, NULL,
-                                          PAGE_READWRITE, 0,
-                                          source->GetVirtualSize(), NULL));
+        PAGE_READWRITE, 0,
+        source->GetVirtualSize(), NULL));
+
+    DWORD allAccess = FILE_MAP_ALL_ACCESS;
+#endif
+
     if (m_FileMap == NULL)
         ThrowLastError();
 
-
-    m_FileView.Assign(CLRMapViewOfFile(m_FileMap, FILE_MAP_ALL_ACCESS, 0, 0, 0,
+    m_FileView.Assign(CLRMapViewOfFile(m_FileMap, allAccess, 0, 0, 0,
                                 (void *) source->GetPreferredBase()));
     if (m_FileView == NULL)
-        m_FileView.Assign(CLRMapViewOfFile(m_FileMap, FILE_MAP_ALL_ACCESS, 0, 0, 0));
+        m_FileView.Assign(CLRMapViewOfFile(m_FileMap, allAccess, 0, 0, 0));
 
     if (m_FileView == NULL)
         ThrowLastError();
 
     source->LayoutILOnly(m_FileView, TRUE); //@TODO should be false for streams
     IfFailThrow(Init(m_FileView));
 
-#ifdef CROSSGEN_COMPILE
+#if defined(CROSSGEN_COMPILE)
     if (HasNativeHeader())
+    {
         ApplyBaseRelocations();
+    }
+#elif !defined(TARGET_UNIX)
+    if (m_isInBundle &&
+        HasCorHeader() &&
+        (HasNativeHeader() || HasReadyToRunHeader()) &&
+        g_fAllowNativeImages)
+    {
+        if (!IsNativeMachineFormat())
+            ThrowHR(COR_E_BADIMAGEFORMAT);
+
+        // Do base relocation for PE, if necessary.
+        // otherwise R2R will be disabled for this image.
+        ApplyBaseRelocations();
+
+        // Check if there is a static function table and install it. (except x86)
+#if !defined(TARGET_X86)
+        COUNT_T cbSize = 0;
+        PT_RUNTIME_FUNCTION   pExceptionDir = (PT_RUNTIME_FUNCTION)GetDirectoryEntryData(IMAGE_DIRECTORY_ENTRY_EXCEPTION, &cbSize);
+        DWORD tableSize = cbSize / sizeof(T_RUNTIME_FUNCTION);
+
+        if (pExceptionDir != NULL)
+        {
+            if (!RtlAddFunctionTable(pExceptionDir, tableSize, (DWORD64)this->GetBase()))
+                ThrowLastError();
+
+            m_pExceptionDir = pExceptionDir;
+        }
+#endif //TARGET_X86
+    }
+#endif
+}
+
+ConvertedImageLayout::~ConvertedImageLayout()
+{
+    CONTRACTL
+    {
+        NOTHROW;
+        GC_TRIGGERS;
+        MODE_ANY;
+    }
+    CONTRACTL_END;
+
+#if !defined(CROSSGEN_COMPILE) && !defined(TARGET_UNIX) && !defined(TARGET_X86)
+    if (m_pExceptionDir)
+    {
+        RtlDeleteFunctionTable(m_pExceptionDir);
+    }
 #endif
 }
 

src/coreclr/src/vm/peimagelayout.h

@@ -53,7 +53,7 @@ class PEImageLayout : public PEDecoder
     static PEImageLayout* LoadFromFlat(PEImageLayout* pflatimage);
     static PEImageLayout* Load(PEImage* pOwner, BOOL bNTSafeLoad, BOOL bThrowOnError = TRUE);
     static PEImageLayout* LoadFlat(PEImage* pOwner);
-    static PEImageLayout* LoadConverted(PEImage* pOwner);
+    static PEImageLayout* LoadConverted(PEImage* pOwner, BOOL isInBundle = FALSE);
     static PEImageLayout* LoadNative(LPCWSTR fullPath);
     static PEImageLayout* Map(PEImage* pOwner);
 #endif
@@ -109,8 +109,12 @@ class ConvertedImageLayout: public PEImageLayout
     CLRMapViewHolder m_FileView;
 public:
 #ifndef DACCESS_COMPILE
-    ConvertedImageLayout(PEImageLayout* source);
+    ConvertedImageLayout(PEImageLayout* source, BOOL isInBundle = FALSE);
+    virtual ~ConvertedImageLayout();
 #endif
+private:
+    bool m_isInBundle;
+    PT_RUNTIME_FUNCTION m_pExceptionDir;
 };
 
 class MappedImageLayout: public PEImageLayout