Generate suppress files automatically for source-code suppressions

docs/user_guide.md

@@ -571,7 +571,7 @@ can be used to "further" enable `alpha.` checkers, and disable `misc` ones.
 
 ~~~~~~~~~~~~~~~~~~~~~
 usage: CodeChecker parse [-h] [-t {plist}] [--suppress SUPPRESS]
-                         [--print-steps]
+                         [--export-source-suppress] [--print-steps]
                          [--verbose {info,debug,debug_analyzer}]
                          [file/folder [file/folder ...]]
 
@@ -596,6 +596,11 @@ optional arguments:
                         to do so.) NOTE: The suppress file relies on the "bug
                         identifier" generated by the analyzers which is
                         experimental, take care when relying on it.
+  --export-source-suppress
+                        Write suppress data from the suppression annotations
+                        found in the source files that were analyzed earlier
+                        that created the results. The suppression information
+                        will be written to the parameter of '--suppress'.
   --print-steps         Print the steps the analyzers took in finding the
                         reported defect. (default: False)
   --verbose {info,debug,debug_analyzer}
@@ -647,7 +652,8 @@ Suppress comments can be used in the source files to suppress specific or all
 checker results found in a particular line of code. Suppress comment should be
 above the line where the defect was found, and no empty lines are allowed
 between the line with the bug and the suppress comment.
-Only comment lines staring with `//` are supported
+
+Only comment lines staring with `//` are supported!
 
 #### Supported formats
 
@@ -679,6 +685,28 @@ void test() {
 }
 ~~~~~~~~~~~~~~~~~~~~~
 
+#### Exporting source code suppression to suppress file
+
+~~~~~~~~~~~~~~~~~~~~~
+  --export-source-suppress
+                        Write suppress data from the suppression annotations
+                        found in the source files that were analyzed earlier
+                        that created the results. The suppression information
+                        will be written to the parameter of '--suppress'.
+~~~~~~~~~~~~~~~~~~~~~
+
+Using `CodeChecker parse`, you can automatically generate a suppress file
+(compatible with `parse`, `store` and `server`) from suppressions found in the
+source code, by specifying `--export-source-suppress` along with a file path in
+`--suppress`.
+
+In case the file given as `--suppress` already exists, it will be extended with
+the results, otherwise, a new file will be created.
+
+~~~~
+CodeChecker parse ./my_plists --suppress generated.suppress --export-source-suppress
+~~~~
+
 ## 4. `store` mode
 
 `store` is used to save previously created machine-readable analysis results

libcodechecker/analyze/analysis_manager.py

@@ -7,6 +7,7 @@
 """
 
 from collections import defaultdict
+import glob
 import multiprocessing
 import os
 import signal
@@ -67,21 +68,13 @@ def worker_result_handler(results, metadata, output_path):
     # as loose files on the disk... but synchronizing LARGE dicts between
     # threads would be more error prone.
     source_map = {}
-    _, _, files = next(os.walk(output_path), ([], [], []))
-    for f in files:
-        if not f.endswith(".source"):
-            continue
-
-        abspath = os.path.join(output_path, f)
-        f = f.replace(".source", '')
-        with open(abspath, 'r') as sfile:
-            source_map[f] = sfile.read().strip()
-
-        os.remove(abspath)
+    for f in glob.glob(os.path.join(output_path, "*.source")):
+        with open(f, 'r') as sfile:
+            source_map[f[:-7]] = sfile.read().strip()
+        os.remove(f)
 
     metadata['result_source_files'] = source_map
 
-
 # Progress reporting.
 progress_checked_num = None
 progress_actions = None
@@ -101,7 +94,8 @@ def check(check_data):
     skiplist handler is None if no skip file was configured.
     """
 
-    action, context, analyzer_config_map, output_dir, skip_handler = check_data
+    action, context, analyzer_config_map, \
+        output_dir, skip_handler = check_data
 
     skipped = False
     try:
@@ -216,7 +210,8 @@ def signal_handler(*arg, **kwarg):
                              context,
                              analyzer_config_map,
                              output_path,
-                             skip_handler) for build_action in actions]
+                             skip_handler)
+                            for build_action in actions]
 
         pool.map_async(check,
                        analyzed_actions,

libcodechecker/analyze/analyzers/result_handler_plist_to_stdout.py

@@ -111,7 +111,16 @@ def __print_bugs(self, reports, files):
                                                                 checker_name)
 
             # Check for suppress comment.
-            if sp_handler.get_suppressed():
+            suppress_data = sp_handler.get_suppressed()
+            if suppress_data:
+                if self.suppress_handler:
+                    LOG.info("Writing source-code suppress at '{0}:{1}' to "
+                             "suppress file".format(source_file, report_line))
+                    hash_value, file_name, comment = suppress_data
+                    self.suppress_handler.store_suppress_bug_id(hash_value,
+                                                                file_name,
+                                                                comment)
+
                 continue
 
             self.__output.write(self.__format_bug_event(checker_name,

libcodechecker/arg_handler.py

@@ -146,7 +146,7 @@ def handle_server(args):
         os.makedirs(workspace)
 
     suppress_handler = generic_package_suppress_handler.\
-        GenericSuppressHandler(None)
+        GenericSuppressHandler(None, False)
     if args.suppress is None:
         LOG.warning('No suppress file was given, suppressed results will '
                     'be only stored in the database.')
@@ -155,6 +155,11 @@ def handle_server(args):
             LOG.error('Suppress file ' + args.suppress + ' not found!')
             sys.exit(1)
 
+        LOG.debug('Using suppress file: ' +
+                  str(suppress_handler.suppress_file))
+        suppress_handler = generic_package_suppress_handler.\
+            GenericSuppressHandler(args.suppress, True)
+
     context = generic_package_context.get_context()
     context.codechecker_workspace = workspace
     session_manager.SessionManager.CodeChecker_Workspace = workspace
@@ -181,9 +186,6 @@ def handle_server(args):
     # Start database viewer.
     db_connection_string = sql_server.get_connection_string()
 
-    suppress_handler.suppress_file = args.suppress
-    LOG.debug('Using suppress file: ' + str(suppress_handler.suppress_file))
-
     checker_md_docs = os.path.join(context.doc_root, 'checker_md_docs')
     checker_md_docs_map = os.path.join(checker_md_docs,
                                        'checker_doc_map.json')

libcodechecker/generic_package_suppress_handler.py

@@ -19,19 +19,26 @@
 
 class GenericSuppressHandler(suppress_handler.SuppressHandler):
 
-    def __init__(self, suppress_file):
+    def __init__(self, suppress_file, allow_write):
         """
         Create a new suppress handler with a suppress_file as backend.
         """
         super(GenericSuppressHandler, self).__init__()
 
         self.__suppress_info = []
+        self.__allow_write = allow_write
+
         if suppress_file:
             self.suppress_file = suppress_file
             self.__have_memory_backend = True
             self.__revalidate_suppress_data()
         else:
             self.__have_memory_backend = False
+            self.__arrow_write = False
+
+            if allow_write:
+                raise ValueError("Can't create allow_write=True suppress "
+                                 "handler without a backend file.")
 
     def __revalidate_suppress_data(self):
         """Reload the information in the suppress file to the memory."""
@@ -47,7 +54,7 @@ def __revalidate_suppress_data(self):
 
     def store_suppress_bug_id(self, bug_id, file_name, comment):
 
-        if self.suppress_file is None:
+        if not self.__allow_write:
             return True
 
         ret = suppress_file_handler.write_to_suppress_file(self.suppress_file,
@@ -59,7 +66,7 @@ def store_suppress_bug_id(self, bug_id, file_name, comment):
 
     def remove_suppress_bug_id(self, bug_id, file_name):
 
-        if self.suppress_file is None:
+        if not self.__allow_write:
             return True
 
         ret = suppress_file_handler.remove_from_suppress_file(

libcodechecker/parse.py

@@ -11,6 +11,7 @@
 import argparse
 import json
 import os
+import sys
 
 from libcodechecker import generic_package_context
 from libcodechecker import generic_package_suppress_handler
@@ -84,6 +85,17 @@ def add_arguments_to_parser(parser):
                              "which is experimental, take care when relying "
                              "on it.")
 
+    parser.add_argument('--export-source-suppress',
+                        dest="create_suppress",
+                        action="store_true",
+                        required=False,
+                        default=argparse.SUPPRESS,
+                        help="Write suppress data from the suppression "
+                             "annotations found in the source files that were "
+                             "analyzed earlier that created the results. "
+                             "The suppression information will be written "
+                             "to the parameter of '--suppress'.")
+
     parser.add_argument('--print-steps',
                         dest="print_steps",
                         action="store_true",
@@ -124,12 +136,12 @@ def parse(f, context, metadata_dict, suppress_handler, steps):
     rh.analyzer_result_file = f
     rh.analyzer_cmd = ""
 
-    rh.analyzed_source_file = "UNKNOWN"
-    base_f = os.path.basename(f)
     if 'result_source_files' in metadata_dict and \
-            base_f in metadata_dict['result_source_files']:
+            f in metadata_dict['result_source_files']:
         rh.analyzed_source_file = \
-            metadata_dict['result_source_files'][base_f]
+            metadata_dict['result_source_files'][f]
+    else:
+        rh.analyzed_source_file = "UNKNOWN"
 
     rh.handle_results()
 
@@ -152,12 +164,28 @@ def main(args):
 
     suppress_handler = None
     if 'suppress' in args:
+        __make_handler = False
         if not os.path.isfile(args.suppress):
-            LOG.warning("Suppress file '" + args.suppress + "' given, but it "
-                        "does not exist -- will not suppress anything.")
+            if 'create_suppress' in args:
+                with open(args.suppress, 'w') as _:
+                    # Just create the file.
+                    __make_handler = True
+                    LOG.info("Will write source-code suppressions to "
+                             "suppress file.")
+            else:
+                LOG.warning("Suppress file '" + args.suppress + "' given, but "
+                            "it does not exist -- will not suppress anything.")
         else:
+            __make_handler = True
+
+        if __make_handler:
             suppress_handler = generic_package_suppress_handler.\
-                GenericSuppressHandler(args.suppress)
+                GenericSuppressHandler(args.suppress,
+                                       'create_suppress' in args)
+    elif 'create_suppress' in args:
+        LOG.error("Can't use '--export-source-suppress' unless '--suppress "
+                  "SUPPRESS_FILE' is also given.")
+        sys.exit(2)
 
     for input_path in args.input:
         os.chdir(original_cwd)

libcodechecker/store.py

@@ -270,15 +270,14 @@ def consume_plist(item):
     rh.analyzer_cmd = ''
 
     rh.analyzed_source_file = "UNKNOWN"
-    base_f = os.path.basename(f)
     if 'result_source_files' in metadata_dict and\
-            base_f in metadata_dict['result_source_files']:
+            f in metadata_dict['result_source_files']:
         rh.analyzed_source_file = \
-            metadata_dict['result_source_files'][base_f]
+            metadata_dict['result_source_files'][f]
     rh.analyzer_result_file = f
 
     if rh.analyzed_source_file == "UNKNOWN":
-        LOG.info("Storing defects in input file '" + base_f + "'")
+        LOG.info("Storing defects in input file '" + f + "'")
     else:
         LOG.info("Storing analysis results for file '" +
                  rh.analyzed_source_file + "'")

tests/functional/suppress/test_files/build.json

@@ -0,0 +1,7 @@
+[
+	{
+		"directory": ".",
+		"command": "g++ suppress.cpp",
+		"file": "suppress.cpp"
+	}
+]

tests/functional/suppress/test_files/expected.suppress

@@ -0,0 +1 @@
+c417110e71a5cb8e8163d524076bb3ed||suppress.cpp||deliberate segfault!

tests/functional/suppress/test_files/suppress.cpp

@@ -0,0 +1,7 @@
+int main()
+{
+    int *p = 0;
+
+    // codechecker_suppress [all] deliberate segfault!
+    return *p + 2;
+}

tests/functional/suppress/test_suppress_generation.py

@@ -0,0 +1,74 @@
+# -----------------------------------------------------------------------------
+#                     The CodeChecker Infrastructure
+#   This file is distributed under the University of Illinois Open Source
+#   License. See LICENSE.TXT for details.
+# -----------------------------------------------------------------------------
+"""
+Test source-code level suppression data writing to suppress file.
+"""
+
+from subprocess import CalledProcessError
+import os
+import shlex
+import subprocess
+import unittest
+
+from libtest import env
+
+
+class TestSuppress(unittest.TestCase):
+    """
+    Test source-code level suppression data writing to suppress file.
+    """
+
+    def setUp(self):
+        self.test_workspace = os.environ['TEST_WORKSPACE']
+        self.test_dir = os.path.join(
+            os.path.dirname(__file__), "test_files")
+
+    def test_source_suppress_export(self):
+        """
+        Test exporting a source suppress comment automatically to file.
+        """
+
+        def __call(command):
+            try:
+                print(' '.join(command))
+                proc = subprocess.Popen(shlex.split(' '.join(command)),
+                                        cwd=self.test_dir,
+                                        stdout=subprocess.PIPE,
+                                        stderr=subprocess.PIPE,
+                                        env=env.test_env())
+                out, err = proc.communicate()
+                print(out)
+                print(err)
+                return 0
+            except CalledProcessError as cerr:
+                print("Failed to call:\n" + ' '.join(cerr.cmd))
+                return cerr.returncode
+
+        analyze_cmd = ['CodeChecker', 'analyze',
+                       os.path.join(self.test_dir, "build.json"),
+                       "--output", os.path.join(self.test_workspace, "reports")
+                       ]
+        ret = __call(analyze_cmd)
+        self.assertEqual(ret, 0, "Couldn't create analysis of test project.")
+
+        generated_file = os.path.join(self.test_workspace,
+                                      "generated.suppress")
+
+        extract_cmd = ['CodeChecker', 'parse',
+                       os.path.join(self.test_workspace, "reports"),
+                       "--suppress", generated_file,
+                       "--export-source-suppress"
+                       ]
+        __call(extract_cmd)
+        self.assertEqual(ret, 0, "Failed to generate suppress file.")
+
+        with open(generated_file, 'r') as generated:
+            with open(os.path.join(self.test_dir, "expected.suppress"),
+                      'r') as expected:
+                self.assertEqual(generated.read().strip(),
+                                 expected.read().strip(),
+                                 "The generated suppress file does not "
+                                 "look like what was expected.")