Speed up installation (RobotLocomotion#7683)

* Speed up installation

* Run `file` command only once, to detect if installed files are executables.
Instead of being run once for each file to test, it is run once with the list
of all the installed files, and returns the list of all installed files with
their file type. The list is then processed in line by line to assess if each
installed file is a binary executable or not. This reduces the installation
time by about 3 to 4s (out of 17-18s)
* Skip testing if a file is an executable if the extension belongs to the
following list: *.h, *.py, *.obj, *.cmake, *.1, *.hpp, *.txt. Since there
are more than 100 files installed for each of these extensions, it decreases
the installation time significantally (by about 3 to 4s).

Installation time is decreased from 16-20s to 8-10s.

tools/install/install.py.in

@@ -8,40 +8,36 @@ import re
 import shutil
 import stat
 import sys
-from subprocess import check_output, check_call, CalledProcessError
+from subprocess import check_output, check_call
 
 subdirs = set()
 prefix = None
-libs = {}
+files_to_fix_up = {}
+potential_binaries = []
 list_only = False
 # On linux, dynamic libraries may have their version number
 # as a suffix (e.g. my_lib.so.x.y.z).
 dylib_match = r"(.*\.so)(\.\d+)*$|(.*\.dylib)$"
 
-def is_binary_executable(dst):
-    """Checks if given executable is an ELF or Mach-O executable
 
-    Returns True iff given executable `dst` is an ELF or Mach-O executable, or
-    an ELF shared object.
+def find_binary_executables():
+    """Finds installed files that are binary executables to fix them up later.
+
+    Takes `potential_binaries` as input list, and updates
+    `files_to_fix_up` with executables that need to be fixed up.
     """
-    # To speed up the process of identifying executables, we skip
-    # header files which should never be executables.
-    if dst.endswith(".h"):
-        return False
     # Checking file type with command `file` is the safest way to find
     # executables. Files without an extension are likely to be executables, but
     # it is not always the case.
-    # TODO(fbudin69500) Explore ways to speed up this process. This could be
-    # achieved by calling the `file` command with multiple input files at once,
-    # if the overhead is in `subprocess.check_output()`.
-    file_output = check_output(["file", dst])
+    file_output = check_output(["file"] + potential_binaries)
     # On Linux, executables can be ELF shared objects.
-    executable_match = r"(.*ELF.*(executable|shared object).*)|(.*Mach-O.*executable.*)"
-    re_result = re.match(executable_match, file_output)
-    if re_result is not None:
-        return True
-    else:
-        return False
+    executable_pattern = r"(.*):.*(ELF.*executable|shared object.*|Mach-O.*executable.*)"  # noqa
+    executable_match = re.compile(executable_pattern)
+    for line in file_output.splitlines():
+        re_result = executable_match.match(line)
+        if re_result is not None:
+            files_to_fix_up[re_result.group(1)] = (re_result.group(1), 1)
+
 
 def needs_install(src, dst):
     # Get canonical destination.
@@ -94,22 +90,34 @@ def install(src, dst):
         # Check that dependency is only referenced once
         # in the library dictionary. If it is referenced multiple times,
         # we do not know which one to use, and fail fast.
-        if basename in libs:
+        if basename in files_to_fix_up:
             sys.stderr.write(
                 "Multiple installation rules found for %s." % (basename))
             sys.exit(1)
-        libs[basename] = (dst_full, installed)
+        files_to_fix_up[basename] = (dst_full, installed)
     else:  # It is not a library, it may be an executable.
-        if is_binary_executable(dst_full):  # It is an executable.
-            libs[dst_full] = (dst_full, installed)
-        else:  # Neither a library nor an executable.
+        # Do not check files with the following extensions (over 100 files
+        # with each extension, and we are certain these are not executables).
+        if (installed and
+                not dst_full.endswith(".h") and
+                not dst_full.endswith(".py") and
+                not dst_full.endswith(".obj") and
+                not dst_full.endswith(".cmake") and
+                not dst_full.endswith(".1") and
+                not dst_full.endswith(".hpp") and
+                not dst_full.endswith(".txt")):
+            potential_binaries.append(dst_full)
+        else:
             pass
 
 
 def fix_rpaths():
+    # Add binary executables to list of files to be fixed up:
+    find_binary_executables()
     # Only fix files that are installed now.
-    fix_libs = [(k, libs[k][0]) for k in libs.keys() if libs[k][1]]
-    for basename, dst_full in fix_libs:
+    fix_files = [(k, files_to_fix_up[k][0])
+                 for k in files_to_fix_up.keys() if files_to_fix_up[k][1]]
+    for basename, dst_full in fix_files:
         # Enable write permissions to allow modification.
         os.chmod(dst_full, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
                  stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
@@ -135,10 +143,11 @@ def macos_fix_rpaths(basename, dst_full):
         dep_basename = os.path.basename(relative_path)
         # Look for the absolute path in the dictionary of fixup files to
         # find library paths.
-        if dep_basename not in libs.keys():
+        if dep_basename not in files_to_fix_up.keys():
             continue
         lib_dirname = os.path.dirname(dst_full)
-        diff_path = os.path.relpath(libs[dep_basename][0], lib_dirname)
+        diff_path = os.path.relpath(files_to_fix_up[dep_basename][0],
+                                    lib_dirname)
         check_call(
             ['install_name_tool',
              "-change", relative_path,
@@ -170,18 +179,18 @@ def linux_fix_rpaths(dst_full):
         ldd_result = line.strip().split(' => ')
         # If library found and not in install prefix, then skip.
         if len(ldd_result) < 2 or \
-                not (ldd_result[1] == "not found"
-                     or ldd_result[1].startswith(prefix)):
+                not (ldd_result[1] == "not found" or
+                     ldd_result[1].startswith(prefix)):
             continue
         re_result = re.match(dylib_match, ldd_result[0])
         # Look for the absolute path in the dictionary of libraries using the
         # library name without its possible version number.
         soname, _, _ = re_result.groups()
-        if soname not in libs.keys():
+        if soname not in files_to_fix_up.keys():
             continue
         lib_dirname = os.path.dirname(dst_full)
         diff_path = os.path.dirname(
-            os.path.relpath(libs[soname][0], lib_dirname)
+            os.path.relpath(files_to_fix_up[soname][0], lib_dirname)
         )
         rpath.append('$ORIGIN' + '/' + diff_path)