Bug 1240848: Adds additional instructions to x64 detour patcher and p…

…revents register clobbering in jmp from trampoline; r=ehsan --HG-- extra : rebase_source : c9604e99a0741213676227cfc7f915236c6e67ee extra : amend_source : 7f2bbe6e661ab4dffc045d24489bf40d0409b093
dothq · Feb 3, 2016 · 33ea2e8 · 33ea2e8
1 parent 0e3cb4b
commit 33ea2e8
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 0 deletions.
diff --git a/toolkit/xre/test/win/TestDllInterceptor.cpp b/toolkit/xre/test/win/TestDllInterceptor.cpp
@@ -159,6 +159,7 @@ int main()
       TestHook("gdi32.dll", "CreateDIBSection") &&
       TestHook("kernel32.dll", "CreateFileW") &&
 #endif
+      TestHook("user32.dll", "CreateWindowExW") &&
       TestHook("imm32.dll", "ImmGetContext") &&
       TestHook("imm32.dll", "ImmGetCompositionStringW") &&
       TestHook("imm32.dll", "ImmSetCandidateWindow") &&

diff --git a/xpcom/build/nsWindowsDllInterceptor.h b/xpcom/build/nsWindowsDllInterceptor.h
@@ -738,6 +738,31 @@ class WindowsDllDetourPatcher
           // not support yet!
           return;
         }
+      } else if (origBytes[nBytes] == 0x66) {
+        // operand override prefix
+        nBytes += 1;
+        // This is the same as the x86 version
+        if (origBytes[nBytes] >= 0x88 && origBytes[nBytes] <= 0x8B) {
+          // various MOVs
+          unsigned char b = origBytes[nBytes + 1];
+          if (((b & 0xc0) == 0xc0) ||
+              (((b & 0xc0) == 0x00) &&
+               ((b & 0x07) != 0x04) && ((b & 0x07) != 0x05))) {
+            // REG=r, R/M=r or REG=r, R/M=[r]
+            nBytes += 2;
+          } else if ((b & 0xc0) == 0x40) {
+            if ((b & 0x07) == 0x04) {
+              // REG=r, R/M=[SIB + disp8]
+              nBytes += 4;
+            } else {
+              // REG=r, R/M=[r + disp8]
+              nBytes += 3;
+            }
+          } else {
+            // complex MOV, bail
+            return;
+          }
+        }
       } else if ((origBytes[nBytes] & 0xf0) == 0x50) {
         // 1-byte push/pop
         nBytes++;
@@ -747,6 +772,9 @@ class WindowsDllDetourPatcher
       } else if (origBytes[nBytes] == 0xb8) {
         // MOV 0xB8: http://ref.x86asm.net/coder32.html#xB8
         nBytes += 5;
+      } else if (origBytes[nBytes] == 0x33) {
+        // xor r32, r/m32
+        nBytes += 2;
       } else if (origBytes[nBytes] == 0xc3) {
         // ret
         nBytes++;