Fix identify_magic() to check that a file that starts with MH_MAGIC is

at least as big as the mach header to be identified as a Mach-O file and
make sure smaller files are not identified as a Mach-O files but as
unknown files. Also fix identify_magic() so it looks at all 4 bytes of
the filetype field when determining the type of the Mach-O file.
Then fix the macho-invalid-header test case to check that it is an
unknown file and make sure it does not get the error for
object_error::parse_failed.  And also update the unit tests.


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@258883 91177308-0d34-0410-b5e6-96231b3b80d8

lib/Support/Path.cpp

@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/COFF.h"
+#include "llvm/Support/MachO.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -1040,12 +1041,24 @@ file_magic identify_magic(StringRef Magic) {
           Magic[2] == char(0xFA) &&
           (Magic[3] == char(0xCE) || Magic[3] == char(0xCF))) {
         /* Native endian */
-        if (Magic.size() >= 16) type = Magic[14] << 8 | Magic[15];
+        size_t MinSize;
+        if (Magic[3] == char(0xCE))
+          MinSize = sizeof(MachO::mach_header);
+        else
+          MinSize = sizeof(MachO::mach_header_64);
+        if (Magic.size() >= MinSize)
+          type = Magic[12] << 24 | Magic[13] << 12 | Magic[14] << 8 | Magic[15];
       } else if ((Magic[0] == char(0xCE) || Magic[0] == char(0xCF)) &&
                  Magic[1] == char(0xFA) && Magic[2] == char(0xED) &&
                  Magic[3] == char(0xFE)) {
         /* Reverse endian */
-        if (Magic.size() >= 14) type = Magic[13] << 8 | Magic[12];
+        size_t MinSize;
+        if (Magic[0] == char(0xCE))
+          MinSize = sizeof(MachO::mach_header);
+        else
+          MinSize = sizeof(MachO::mach_header_64);
+        if (Magic.size() >= MinSize)
+          type = Magic[15] << 24 | Magic[14] << 12 |Magic[13] << 8 | Magic[12];
       }
       switch (type) {
         default: break;

test/Object/macho-invalid.test

@@ -56,7 +56,8 @@ RUN:      | FileCheck -check-prefix INVALID-SECTION-IDX-SYMBOL-SEC-pax %s
 INVALID-SECTION-IDX-SYMBOL-SEC-pax: 0000000100000000 0f 42 0010 00000065 __mh_execute_header
 
 RUN: not llvm-objdump -private-headers %p/Inputs/macho-invalid-header 2>&1 | FileCheck -check-prefix INVALID-HEADER %s
-INVALID-HEADER: Invalid data was encountered while parsing the file
+INVALID-HEADER: The file was not recognized as a valid object file.
+NOT-INVALID-HEADER: Invalid data was encountered while parsing the file.
 
 RUN: not llvm-objdump -private-headers %p/Inputs/macho64-invalid-incomplete-segment-load-command 2>&1 | FileCheck -check-prefix INCOMPLETE-SEGMENT-LOADC %s
 INCOMPLETE-SEGMENT-LOADC: Invalid data was encountered while parsing the file

unittests/Support/Path.cpp

@@ -726,21 +726,29 @@ const char coff_import_library[] = "\x00\x00\xff\xff....";
 const char elf_relocatable[] = { 0x7f, 'E', 'L', 'F', 1, 2, 1, 0, 0,
                                  0,    0,   0,   0,   0, 0, 0, 0, 1 };
 const char macho_universal_binary[] = "\xca\xfe\xba\xbe...\0x00";
-const char macho_object[] = "\xfe\xed\xfa\xce..........\x00\x01";
-const char macho_executable[] = "\xfe\xed\xfa\xce..........\x00\x02";
+const char macho_object[] =
+    "\xfe\xed\xfa\xce........\x00\x00\x00\x01............";
+const char macho_executable[] =
+    "\xfe\xed\xfa\xce........\x00\x00\x00\x02............";
 const char macho_fixed_virtual_memory_shared_lib[] =
-    "\xfe\xed\xfa\xce..........\x00\x03";
-const char macho_core[] = "\xfe\xed\xfa\xce..........\x00\x04";
-const char macho_preload_executable[] = "\xfe\xed\xfa\xce..........\x00\x05";
+    "\xfe\xed\xfa\xce........\x00\x00\x00\x03............";
+const char macho_core[] =
+    "\xfe\xed\xfa\xce........\x00\x00\x00\x04............";
+const char macho_preload_executable[] =
+    "\xfe\xed\xfa\xce........\x00\x00\x00\x05............";
 const char macho_dynamically_linked_shared_lib[] =
-    "\xfe\xed\xfa\xce..........\x00\x06";
-const char macho_dynamic_linker[] = "\xfe\xed\xfa\xce..........\x00\x07";
-const char macho_bundle[] = "\xfe\xed\xfa\xce..........\x00\x08";
-const char macho_dsym_companion[] = "\xfe\xed\xfa\xce..........\x00\x0a";
-const char macho_kext_bundle[] = "\xfe\xed\xfa\xce..........\x00\x0b";
+    "\xfe\xed\xfa\xce........\x00\x00\x00\x06............";
+const char macho_dynamic_linker[] =
+    "\xfe\xed\xfa\xce........\x00\x00\x00\x07............";
+const char macho_bundle[] =
+    "\xfe\xed\xfa\xce........\x00\x00\x00\x08............";
+const char macho_dsym_companion[] =
+    "\xfe\xed\xfa\xce........\x00\x00\x00\x0a............";
+const char macho_kext_bundle[] =
+    "\xfe\xed\xfa\xce........\x00\x00\x00\x0b............";
 const char windows_resource[] = "\x00\x00\x00\x00\x020\x00\x00\x00\xff";
 const char macho_dynamically_linked_shared_lib_stub[] =
-    "\xfe\xed\xfa\xce..........\x00\x09";
+    "\xfe\xed\xfa\xce........\x00\x00\x00\x09............";
 
 TEST_F(FileSystemTest, Magic) {
   struct type {