Implement Emscripten toolchain profiler which can be used to profile …

…the process-wide execution of the toolchain in action.

em++

@@ -3,14 +3,18 @@
 # This script should work in python 2 *or* 3. It loads emcc.py, which needs python 2.
 # It also tells emcc.py that we want C++ and not C by default
 
+from tools.toolchain_profiler import ToolchainProfiler
+
 import sys
 
 sys.argv += ['--emscripten-cxx']
 
 if sys.version_info.major == 2:
   import emcc
   if __name__ == '__main__':
+    ToolchainProfiler.record_process_start()
     emcc.run()
+    sys.exit(0)
 else:
   import os, subprocess
   if __name__ == '__main__':

emar

@@ -2,15 +2,16 @@
 
 # This script should work in python 2 *or* 3. It loads emar.py, which needs python 2.
 
+from tools.toolchain_profiler import ToolchainProfiler
 
 import sys
 
-
-
 if sys.version_info.major == 2:
   import emar
   if __name__ == '__main__':
+    ToolchainProfiler.record_process_start()
     emar.run()
+    sys.exit(0)
 else:
   import os, subprocess
   if __name__ == '__main__':

emar.py

@@ -7,6 +7,8 @@
 This script acts as a frontend replacement for ar. See emcc.
 '''
 
+from tools.toolchain_profiler import ToolchainProfiler
+
 import os, subprocess, sys
 from tools import shared
 
@@ -59,4 +61,6 @@ def run():
       shared.try_delete(d)
 
 if __name__ == '__main__':
+  ToolchainProfiler.record_process_start()
   run()
+  sys.exit(0)

emcc

@@ -2,15 +2,16 @@
 
 # This script should work in python 2 *or* 3. It loads emcc.py, which needs python 2.
 
+from tools.toolchain_profiler import ToolchainProfiler
 
 import sys
 
-
-
 if sys.version_info.major == 2:
   import emcc
   if __name__ == '__main__':
+    ToolchainProfiler.record_process_start()
     emcc.run()
+    sys.exit(0)
 else:
   import os, subprocess
   if __name__ == '__main__':

emcc.py

@@ -23,6 +23,10 @@
   EMMAKEN_COMPILER - The compiler to be used, if you don't want the default clang.
 '''
 
+from tools.toolchain_profiler import ToolchainProfiler
+if __name__ == '__main__':
+  ToolchainProfiler.record_process_start()
+
 import os, sys, shutil, tempfile, subprocess, shlex, time, re, logging
 from subprocess import PIPE
 from tools import shared, jsrun, system_libs
@@ -2103,3 +2107,4 @@ def un_src(): # use this if you want to modify the script and need it to be inli
 
 if __name__ == '__main__':
   run()
+  sys.exit(0)

emprofile.bat

@@ -0,0 +1,2 @@
+@echo off
+python "%~dp0\emprofile.py" %*

emprofile.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+
+import sys, shutil, os, json, tempfile, time
+
+profiler_logs_path = os.path.join(tempfile.gettempdir(), 'emscripten_toolchain_profiler_logs')
+
+# If set to 1, always generates the output file under the same filename and doesn't delete the temp data.
+DEBUG_EMPROFILE_PY = 0
+
+OUTFILE = 'toolchain_profiler.results_' + time.strftime('%Y%m%d_%H%M')
+for arg in sys.argv:
+  if arg.startswith('--outfile='):
+    OUTFILE = arg.split('=')[1].strip().replace('.html', '')
+
+# Deletes all previously captured log files to make room for a new clean run.
+def delete_profiler_logs():
+  try:
+    shutil.rmtree(profiler_logs_path)
+  except:
+    pass
+
+def list_files_in_directory(d):
+  files = []
+  try:
+    items = os.listdir(d)
+    for i in items:
+      f = os.path.join(d, i)
+      if os.path.isfile(f):
+        files += [f]
+    return files
+  except:
+    return []
+
+def create_profiling_graph():
+  log_files = [f for f in list_files_in_directory(profiler_logs_path) if 'toolchain_profiler.pid_' in f]
+
+  all_results = []
+  if len(log_files) > 0:
+    print 'Processing ' + str(len(log_files)) + ' profile log files in "' + profiler_logs_path + '"...'
+  for f in log_files:
+    try:
+      json_data = open(f, 'r').read()
+      lines = json_data.split('\n')
+      lines = filter(lambda x: x != '[' and x != ']' and x != ',' and len(x.strip()) > 0, lines)
+      lines = map(lambda x: (x + ',') if not x.endswith(',') else x, lines)
+      lines[-1] = lines[-1][:-1]
+      json_data = '[' + '\n'.join(lines) + ']'
+      all_results += json.loads(json_data)
+    except Exception, e:
+      print >> sys.stderr, str(e)
+      print >> sys.stderr, 'Failed to parse JSON file "' + f + '"!'
+      sys.exit(1)
+  if len(all_results) == 0:
+    print 'No profiler logs were found in path "' + profiler_logs_path + '". Try setting the environment variable EM_PROFILE_TOOLCHAIN=1 and run some emcc commands, and then rerun "python emprofile.py --graph" again.'
+    return
+
+  all_results.sort(key=lambda x: x['time'])
+
+  json_file = OUTFILE + '.json'
+  open(json_file, 'w').write(json.dumps(all_results, indent=2))
+  print 'Wrote "' + json_file + '"'
+
+  html_file = OUTFILE + '.html'
+  html_contents = open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'tools', 'toolchain_profiler.results_template.html'), 'r').read().replace('{{{results_log_file}}}', '"' + json_file + '"')
+  open(html_file, 'w').write(html_contents)
+  print 'Wrote "' + html_file + '"'
+
+  if not DEBUG_EMPROFILE_PY:
+    delete_profiler_logs()
+
+if len(sys.argv) < 2:
+  print '''Usage:
+       emprofile.py --reset
+         Deletes all previously recorded profiling log files.
+
+       emprofile.py --graph
+         Draws a graph from all recorded profiling log files.
+
+Optional parameters:
+
+        --outfile=x.html
+          Specifies the name of the results file to generate.
+'''
+  sys.exit(1)
+
+if '--reset' in sys.argv:
+  delete_profiler_logs()
+elif '--graph' in sys.argv:
+  create_profiling_graph()
+else:
+  print 'Unknown command "' + sys.argv[1] + '"!'
+  sys.exit(1)

emscripten.py

@@ -9,6 +9,10 @@
 headers, for the libc implementation in JS).
 '''
 
+from tools.toolchain_profiler import ToolchainProfiler
+if __name__ == '__main__':
+  ToolchainProfiler.record_process_start()
+
 import os, sys, json, optparse, subprocess, re, time, logging
 
 from tools import shared
@@ -1729,3 +1733,4 @@ def _main(args=None):
 
 if __name__ == '__main__':
   _main()
+  sys.exit(0)

tools/js_optimizer.py

@@ -1,4 +1,8 @@
 
+from toolchain_profiler import ToolchainProfiler
+if __name__ == '__main__':
+  ToolchainProfiler.record_process_start()
+
 import os, sys, subprocess, multiprocessing, re, string, json, shutil, logging
 import shared
 
@@ -559,4 +563,5 @@ def run(filename, passes, js_engine=shared.NODE_JS, source_map=False, extra_info
     extra_info = None
   out = run(sys.argv[1], sys.argv[2:], extra_info=extra_info)
   shutil.copyfile(out, sys.argv[1] + '.jsopt.js')
+  sys.exit(0)
 

tools/jsrun.py

@@ -1,3 +1,4 @@
+from toolchain_profiler import ToolchainProfiler
 import time, os, sys, logging
 from subprocess import Popen, PIPE, STDOUT
 

tools/shared.py

@@ -1,3 +1,4 @@
+from toolchain_profiler import ToolchainProfiler
 import shutil, time, os, sys, json, tempfile, copy, shlex, atexit, subprocess, hashlib, cPickle, re, errno
 from subprocess import Popen, PIPE, STDOUT
 from tempfile import mkstemp