Merge pull request Ericsson#2039 from dkrupp/compiler_info

Resurrect --compiler-info-file analyze flag.

analyzer/codechecker_analyzer/buildlog/log_parser.py

@@ -19,6 +19,7 @@
 import traceback
 
 from codechecker_common.logger import get_logger
+from codechecker_common.util import load_json_or_empty
 
 from .. import gcc_toolchain
 from .build_action import BuildAction
@@ -434,21 +435,39 @@ def get_compiler_standard(compiler, language):
         return standard
 
     @staticmethod
-    def set(details):
+    def load_compiler_info(filename, compiler):
+        contents = load_json_or_empty(filename, {})
+        compiler_info = contents.get(compiler)
+        if compiler_info is None:
+            LOG.error("Could not find compiler %s in file %s",
+                      compiler, filename)
+
         ICI = ImplicitCompilerInfo
+        ICI.compiler_includes[compiler] = compiler_info.get('includes')
+        ICI.compiler_standard[compiler] = compiler_info.get('default_standard')
+        ICI.compiler_target[compiler] = compiler_info.get('target')
 
-        if details['compiler'] not in ICI.compiler_includes:
-            ICI.compiler_includes[details['compiler']] = \
-                ICI.get_compiler_includes(details['compiler'],
-                                          details['lang'],
-                                          details['analyzer_options'])
-        if details['compiler'] not in ICI.compiler_target:
-            ICI.compiler_target[details['compiler']] = \
-                ICI.get_compiler_target(details['compiler'])
-        if details['compiler'] not in ICI.compiler_standard:
-            ICI.compiler_standard[details['compiler']] = \
-                ICI.get_compiler_standard(details['compiler'],
-                                          details['lang'])
+    @staticmethod
+    def set(details, compiler_info_file=None):
+        ICI = ImplicitCompilerInfo
+
+        if compiler_info_file and os.path.exists(compiler_info_file):
+            # Compiler info file exists, load it.
+            ICI.load_compiler_info(compiler_info_file, details['compiler'])
+        else:
+            # Invoke compiler to gather implicit compiler info.
+            if details['compiler'] not in ICI.compiler_includes:
+                ICI.compiler_includes[details['compiler']] = \
+                    ICI.get_compiler_includes(details['compiler'],
+                                              details['lang'],
+                                              details['analyzer_options'])
+            if details['compiler'] not in ICI.compiler_standard:
+                ICI.compiler_standard[details['compiler']] = \
+                    ICI.get_compiler_standard(details['compiler'],
+                                              details['lang'])
+            if details['compiler'] not in ICI.compiler_target:
+                ICI.compiler_target[details['compiler']] = \
+                    ICI.get_compiler_target(details['compiler'])
 
         details['compiler_includes'] = details['compiler_includes'] or \
             ICI.compiler_includes[details['compiler']]
@@ -511,7 +530,7 @@ def determine_compiler(gcc_command):
     """
     This function determines the compiler from the given compilation command.
     If the first part of the gcc_command is ccache invocation then the rest
-    should be a compilete compilation command.
+    should be a complete compilation command.
 
     CCache may have two forms:
     1. ccache g++ main.cpp
@@ -699,7 +718,7 @@ def f(flag_iterator, details):
     return f
 
 
-def parse_options(compilation_db_entry):
+def parse_options(compilation_db_entry, compiler_info_file=None):
     """
     This function parses a GCC compilation action and returns a BuildAction
     object which can be the input of Clang analyzer tools.
@@ -708,6 +727,7 @@ def parse_options(compilation_db_entry):
                             file, i.e. a dictionary with the compilation
                             command, the compiled file and the current working
                             directory.
+    compiler_info_file -- Contains the path to a compiler info file.
     """
 
     details = {
@@ -795,7 +815,7 @@ def parse_options(compilation_db_entry):
 
     # Store the compiler built in include paths and defines.
     if not toolchain and 'ccache' not in details['compiler']:
-        ImplicitCompilerInfo.set(details)
+        ImplicitCompilerInfo.set(details, compiler_info_file)
 
     return BuildAction(**details)
 
@@ -818,25 +838,36 @@ class CompileActionUniqueingType(object):
 
 
 def parse_unique_log(compilation_database,
+                     report_dir,
                      compile_uniqueing="none",
                      skip_handler=None,
                      compiler_info_file=None):
     """
+    This function reads up the compilation_database
+    and returns with a list of build actions that is prepared for clang
+    execution. That means that gcc specific parameters are filtered out
+    and gcc built in targets and include paths are added.
+    It also filters out duplicate compilation actions based on the
+    compile_uniqueing parameter.
+    This function also dumps auto-detected the compiler info
+    into <report_dir>/compiler_info.json.
+
     compilation_database -- A compilation database as a list of dict objects.
                             These object should contain "file", "dictionary"
                             and "command" keys. The "command" may be replaced
                             by "arguments" which is a split command. Older
                             versions of intercept-build provide the build
                             command this way.
+    report_dir  -- The output report directory. The compiler infos
+                   will be written to <report_dir>/compiler.info.json.
     compile_uniqueing -- Compilation database uniqueing mode.
                          If there are more than one compile commands for a
                          target file, only a single one is kept.
     skip_handler -- A SkipListHandler object which helps to skip build actions
                     that shouldn't be analyzed. The build actions described by
                     this handler will not be the part of the result list.
-    compiler_info_file -- An optional filename where implicit compiler data
-                          is dumped (implicit include paths, architecture
-                          targets, default standard version).
+    compiler_info_file -- compiler_info.json. If exists, it will be used for
+                    analysis.
     """
     try:
         uniqued_build_actions = dict()
@@ -855,9 +886,7 @@ def parse_unique_log(compilation_database,
             if skip_handler and skip_handler.should_skip(entry['file']):
                 LOG.debug("SKIPPING FILE %s", entry['file'])
                 continue
-
-            action = parse_options(entry)
-            LOG.debug(action)
+            action = parse_options(entry, compiler_info_file)
 
             if not action.lang:
                 continue
@@ -903,10 +932,10 @@ def parse_unique_log(compilation_database,
                               compile_uniqueing)
                     sys.exit(1)
 
-        # Filter out duplicate compilation commands.
-        if compiler_info_file:
-            with open(compiler_info_file, 'w') as f:
-                json.dump(ImplicitCompilerInfo.get(), f)
+        compiler_info_out = os.path.join(report_dir, "compiler_info.json")
+        with open(compiler_info_out, 'w') as f:
+            LOG.debug("Writing compiler info into:"+compiler_info_out)
+            json.dump(ImplicitCompilerInfo.get(), f)
 
         LOG.debug('Parsing log file done.')
         return list(uniqued_build_actions.values())

analyzer/codechecker_analyzer/cmd/analyze.py

@@ -124,6 +124,14 @@ def add_arguments_to_parser(parser):
                         default=argparse.SUPPRESS,
                         help="Store the analysis output in the given folder.")
 
+    parser.add_argument('--compiler-info-file',
+                        dest="compiler_info_file",
+                        required=False,
+                        default=argparse.SUPPRESS,
+                        help="Read the compiler includes and target from the "
+                             "specified file rather than invoke the compiler "
+                             "executable.")
+
     parser.add_argument('-t', '--type', '--output-format',
                         dest="output_format",
                         required=False,
@@ -549,6 +557,17 @@ def main(args):
     if 'none' in args.compile_uniqueing and 'ctu_phases' in args:
         args.compile_uniqueing = "alpha"
 
+    compiler_info_file = None
+    if 'compiler_info_file' in args:
+        LOG.debug("Compiler info is read from: %s", args.compiler_info_file)
+        if not os.path.exists(args.compiler_info_file):
+            LOG.error("Compiler info file %s does not exist",
+                      args.compiler_info_file)
+            sys.exit(1)
+        compiler_info_file = args.compiler_info_file
+
+    report_dir = args.output_path
+
     # Parse the JSON CCDBs and retrieve the compile commands.
     actions = []
     for log_file in args.logfile:
@@ -559,10 +578,12 @@ def main(args):
 
         actions += log_parser.parse_unique_log(
             load_json_or_empty(log_file),
+            report_dir,
             args.compile_uniqueing,
             skip_handler,
-            os.path.join(args.output_path, 'compiler_info.json')
+            compiler_info_file
             )
+
     if not actions:
         LOG.info("None of the specified build log files contained "
                  "valid compilation commands. No analysis needed...")

analyzer/tests/functional/analyze/test_analyze.py

@@ -154,6 +154,52 @@ def test_compiler_info_files(self):
                 self.fail("json.load should successfully parse the file %s"
                           % info_File)
 
+    def test_compiler_info_file_is_loaded(self):
+        '''
+        Test that compiler info file is loaded if option is set.
+        '''
+        reports_dir = self.report_dir
+        build_json = os.path.join(self.test_workspace, "build_simple.json")
+        source_file = os.path.join(self.test_workspace, "simple.cpp")
+        compiler_info_file = os.path.join(self.test_workspace,
+                                          "compiler_info.json")
+
+        # Create a compilation database.
+        build_log = [{"directory": self.test_workspace,
+                      "command": "clang++ -c " + source_file,
+                      "file": source_file}]
+
+        with open(build_json, 'w') as outfile:
+            json.dump(build_log, outfile)
+
+        # Test file contents
+        simple_file_content = "int main() { return 0; }"
+
+        # Write content to the test file
+        with open(source_file, 'w') as source:
+            source.write(simple_file_content)
+
+        with open(compiler_info_file, 'w') as source:
+            source.write('{"clang++": {"default_standard": "-std=FAKE_STD", '
+                         '"target": "FAKE_TARGET", "includes": ["-isystem '
+                         '/FAKE_INCLUDE_DIR"]}}')
+
+        # Create analyze command.
+        analyze_cmd = [self._codechecker_cmd, "analyze", build_json,
+                       "--compiler-info-file", compiler_info_file,
+                       "--analyzers", "clangsa", "--verbose", "debug",
+                       "-o", reports_dir]
+
+        # Run analyze.
+        process = subprocess.Popen(
+            analyze_cmd, stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE, cwd=self.test_dir)
+        out, _ = process.communicate()
+
+        self.assertTrue("-std=FAKE_STD" in out)
+        self.assertTrue("--target=FAKE_TARGET" in out)
+        self.assertTrue("-isystem /FAKE_INCLUDE_DIR" in out)
+
     def test_capture_analysis_output(self):
         """
         Test if reports/success/<output_file>.[stdout,stderr].txt

analyzer/tests/unit/test_buildcmd_escaping.py

@@ -92,7 +92,7 @@ def test_analyzer_exec_double_quote(self):
             ' -DDEBUG \'-DMYPATH="/this/some/path/"\''
 
         comp_actions = log_parser.\
-            parse_unique_log(self.__get_cmp_json(compile_cmd))
+            parse_unique_log(self.__get_cmp_json(compile_cmd), self.tmp_dir)
 
         for comp_action in comp_actions:
             cmd = [self.compiler]
@@ -118,7 +118,7 @@ def test_analyzer_ansic_double_quote(self):
         """
         compile_cmd = self.compiler + ''' '-DMYPATH=\"/some/other/path\"' '''
         comp_actions = log_parser.\
-            parse_unique_log(self.__get_cmp_json(compile_cmd))
+            parse_unique_log(self.__get_cmp_json(compile_cmd), self.tmp_dir)
 
         for comp_action in comp_actions:
             cmd = [self.compiler]

analyzer/tests/unit/test_log_parser.py

@@ -51,7 +51,7 @@ def test_old_ldlogger(self):
         # define being considered a file and ignored, for now.
 
         build_action = log_parser.\
-            parse_unique_log(load_json_or_empty(logfile))[0]
+            parse_unique_log(load_json_or_empty(logfile), self.__this_dir)[0]
 
         self.assertEqual(build_action.source, r'/tmp/a.cpp')
         self.assertEqual(len(build_action.analyzer_options), 1)
@@ -70,7 +70,7 @@ def test_new_ldlogger(self):
         # and --target=x86_64-linux-gnu.
 
         build_action = log_parser.\
-            parse_unique_log(load_json_or_empty(logfile))[0]
+            parse_unique_log(load_json_or_empty(logfile), self.__this_dir)[0]
 
         self.assertEqual(build_action.source, r'/tmp/a.cpp')
         self.assertEqual(len(build_action.analyzer_options), 1)
@@ -82,7 +82,7 @@ def test_new_ldlogger(self):
         logfile = os.path.join(self.__test_files, "ldlogger-new-space.json")
 
         build_action = log_parser.\
-            parse_unique_log(load_json_or_empty(logfile))[0]
+            parse_unique_log(load_json_or_empty(logfile), self.__this_dir)[0]
 
         self.assertEqual(build_action.source, r'/tmp/a b.cpp')
         self.assertEqual(build_action.lang, 'c++')
@@ -99,7 +99,7 @@ def test_old_intercept_build(self):
         # The define is passed to the analyzer properly.
 
         build_action = log_parser.\
-            parse_unique_log(load_json_or_empty(logfile))[0]
+            parse_unique_log(load_json_or_empty(logfile), self.__this_dir)[0]
 
         self.assertEqual(build_action.source, r'/tmp/a.cpp')
         self.assertEqual(len(build_action.analyzer_options), 1)
@@ -111,7 +111,7 @@ def test_old_intercept_build(self):
         logfile = os.path.join(self.__test_files, "intercept-old-space.json")
 
         build_action = log_parser.\
-            parse_unique_log(load_json_or_empty(logfile))[0]
+            parse_unique_log(load_json_or_empty(logfile), self.__this_dir)[0]
 
         self.assertEqual(build_action.source, '/tmp/a b.cpp')
         self.assertEqual(build_action.lang, 'c++')
@@ -132,7 +132,7 @@ def test_new_intercept_build(self):
         # The define is passed to the analyzer properly.
 
         build_action = log_parser.\
-            parse_unique_log(load_json_or_empty(logfile))[0]
+            parse_unique_log(load_json_or_empty(logfile), self.__this_dir)[0]
 
         self.assertEqual(build_action.source, r'/tmp/a.cpp')
         self.assertEqual(len(build_action.analyzer_options), 1)
@@ -144,7 +144,7 @@ def test_new_intercept_build(self):
         logfile = os.path.join(self.__test_files, "intercept-new-space.json")
 
         build_action = log_parser.\
-            parse_unique_log(load_json_or_empty(logfile))[0]
+            parse_unique_log(load_json_or_empty(logfile), self.__this_dir)[0]
 
         self.assertEqual(build_action.source, '/tmp/a b.cpp')
         self.assertEqual(build_action.lang, 'c++')
@@ -173,7 +173,8 @@ def test_omit_preproc(self):
              "command": "g++ /tmp/a.cpp -M /tmp/a.cpp",
              "file": "/tmp/a.cpp"}]
 
-        build_actions = log_parser.parse_unique_log(preprocessor_actions)
+        build_actions = log_parser.parse_unique_log(preprocessor_actions,
+                                                    self.__this_dir)
         self.assertEqual(len(build_actions), 1)
         self.assertTrue('-M' not in build_actions[0].original_command)
         self.assertTrue('-E' not in build_actions[0].original_command)
@@ -188,7 +189,8 @@ def test_keep_compile_and_dep(self):
              "command": "g++ /tmp/a.cpp -MD /tmp/a.cpp",
              "file": "/tmp/a.cpp"}]
 
-        build_actions = log_parser.parse_unique_log(preprocessor_actions)
+        build_actions = log_parser.parse_unique_log(preprocessor_actions,
+                                                    self.__this_dir)
         self.assertEqual(len(build_actions), 1)
         self.assertTrue('-MD' in build_actions[0].original_command)
 
@@ -203,7 +205,8 @@ def test_omit_dep_with_e(self):
              "command": "g++ /tmp/a.cpp -E -MD /tmp/a.cpp",
              "file": "/tmp/a.cpp"}]
 
-        build_actions = log_parser.parse_unique_log(preprocessor_actions)
+        build_actions = log_parser.parse_unique_log(preprocessor_actions,
+                                                    self.__this_dir)
         self.assertEqual(len(build_actions), 0)
 
     def test_include_rel_to_abs(self):
@@ -213,7 +216,7 @@ def test_include_rel_to_abs(self):
         logfile = os.path.join(self.__test_files, "include.json")
 
         build_action = log_parser.\
-            parse_unique_log(load_json_or_empty(logfile))[0]
+            parse_unique_log(load_json_or_empty(logfile), self.__this_dir)[0]
 
         self.assertEqual(len(build_action.analyzer_options), 4)
         self.assertEqual(build_action.analyzer_options[0], '-I')