SERVER-65781 Run the symbolizer on CI output

buildscripts/build_system_options.py

@@ -9,7 +9,7 @@ class PathOptions:
     shared_library_folder_name = "lib"
 
     # extend the below list if there are new types of shared libraries
-    _shared_library_file_patterns = [r".+\.so(\.\d{1,})?$", r".+\.dylib$"]
+    _shared_library_file_patterns = [r".+\.so(\.(\d{1,}|debug))?$", r".+\.dylib$"]
     _compiled_shared_library_file_patterns = None
 
     @property

buildscripts/idl/check_stable_api_commands_have_idl_definitions.py

@@ -96,16 +96,16 @@ def list_commands_for_api(api_version: str, mongod_or_mongos: str, install_dir:
     if mongod_or_mongos == "mongod":
         logger = loggers.new_fixture_logger("MongoDFixture", 0)
         logger.parent = LOGGER
-        fixture: interface.Fixture = fixturelib.make_fixture("MongoDFixture", logger, 0,
-                                                             dbpath_prefix=dbpath.name,
-                                                             mongod_executable=mongod_executable)
+        fixture: interface.Fixture = fixturelib.make_fixture(
+            "MongoDFixture", logger, 0, dbpath_prefix=dbpath.name,
+            mongod_executable=mongod_executable, mongod_options={"set_parameters": {}})
     else:
         logger = loggers.new_fixture_logger("ShardedClusterFixture", 0)
         logger.parent = LOGGER
-        fixture = fixturelib.make_fixture("ShardedClusterFixture", logger, 0,
-                                          dbpath_prefix=dbpath.name,
-                                          mongos_executable=mongos_executable,
-                                          mongod_executable=mongod_executable, mongod_options={})
+        fixture = fixturelib.make_fixture(
+            "ShardedClusterFixture", logger, 0, dbpath_prefix=dbpath.name,
+            mongos_executable=mongos_executable, mongod_executable=mongod_executable,
+            mongod_options={"set_parameters": {}})
 
     fixture.setup()
     fixture.await_ready()

buildscripts/resmokelib/config.py

@@ -576,3 +576,12 @@ def all_options(cls):
 # that get loaded for multiversion tests can behave correctly on master and on v5.0; the latter
 # case runs 5.0 and 4.4 binaries and has this value set to True. Can be removed after 6.0.
 USE_LEGACY_MULTIVERSION = True
+
+# Expansions file location
+# in CI, the expansions file is located in the ${workdir}, one dir up
+# from src, the checkout directory
+EXPANSIONS_FILE = "../expansions.yml" if 'CI' in os.environ else None
+
+# Symbolizer secrets
+SYMBOLIZER_CLIENT_SECRET = None
+SYMBOLIZER_CLIENT_ID = None

buildscripts/resmokelib/configure_resmoke.py

@@ -27,6 +27,7 @@ def validate_and_update_config(parser, args):
     """Validate inputs and update config module."""
     _validate_options(parser, args)
     _update_config_vars(args)
+    _update_symbolizer_secrets()
     _validate_config(parser)
     _set_logging_config()
 
@@ -502,3 +503,13 @@ def _tags_from_list(tags_list):
             tags.extend([t for t in tag.split(",") if t != ""])
         return tags
     return None
+
+
+def _update_symbolizer_secrets():
+    """Open `expansions.yml`, get values for symbolizer secrets and update their values inside config.py ."""
+    if not _config.EVERGREEN_TASK_ID:
+        # not running on Evergreen
+        return
+    yml_data = utils.load_yaml_file(_config.EXPANSIONS_FILE)
+    _config.SYMBOLIZER_CLIENT_SECRET = yml_data.get("symbolizer_client_secret")
+    _config.SYMBOLIZER_CLIENT_ID = yml_data.get("symbolizer_client_id")

buildscripts/resmokelib/testing/fixtures/standalone.py

@@ -4,6 +4,8 @@
 import os.path
 import time
 import shutil
+import uuid
+
 import yaml
 
 import pymongo
@@ -23,6 +25,9 @@ def __init__(  # pylint: disable=too-many-arguments
         self.mongod_options = self.fixturelib.make_historic(
             self.fixturelib.default_if_none(mongod_options, {}))
 
+        if "set_parameters" not in self.mongod_options:
+            self.mongod_options["set_parameters"] = {}
+
         if add_feature_flags:
             for ff in self.config.ENABLED_FEATURE_FLAGS:
                 self.mongod_options["set_parameters"][ff] = "true"
@@ -52,6 +57,11 @@ def __init__(  # pylint: disable=too-many-arguments
         self.port = port or fixturelib.get_next_port(job_num)
         self.mongod_options["port"] = self.port
 
+        # Always log backtraces to a file in the dbpath in our testing.
+        backtrace_log_file_name = os.path.join(self.get_dbpath_prefix(),
+                                               uuid.uuid4().hex + ".stacktrace")
+        self.mongod_options["set_parameters"]["backtraceLogFile"] = backtrace_log_file_name
+
     def setup(self):
         """Set up the mongod."""
         if not self.preserve_dbpath and os.path.lexists(self._dbpath):
@@ -61,7 +71,8 @@ def setup(self):
 
         launcher = MongodLauncher(self.fixturelib)
         # Second return val is the port, which we ignore because we explicitly created the port above.
-        # The port is used to set other mongod_option's here: https://github.com/mongodb/mongo/blob/532a6a8ae7b8e7ab5939e900759c00794862963d/buildscripts/resmokelib/testing/fixtures/replicaset.py#L136
+        # The port is used to set other mongod_option's here:
+        # https://github.com/mongodb/mongo/blob/532a6a8ae7b8e7ab5939e900759c00794862963d/buildscripts/resmokelib/testing/fixtures/replicaset.py#L136
         mongod, _ = launcher.launch_mongod_program(self.logger, self.job_num,
                                                    executable=self.mongod_executable,
                                                    mongod_options=self.mongod_options)
@@ -363,3 +374,7 @@ def _add_testing_set_parameters(suite_set_parameters):
     """
     suite_set_parameters.setdefault("testingDiagnosticsEnabled", True)
     suite_set_parameters.setdefault("enableTestCommands", True)
+    # The exact file location is on a per-process basis, so it'll have to be determined when the process gets spun up.
+    # Set it to true for now as a placeholder that will error if no further processing is done.
+    # The placeholder is needed so older versions don't have this option won't have this value set.
+    suite_set_parameters.setdefault("backtraceLogFile", True)

buildscripts/resmokelib/testing/report.py

@@ -10,6 +10,7 @@
 
 from buildscripts.resmokelib import config as _config
 from buildscripts.resmokelib import logging
+from buildscripts.resmokelib.testing.symbolizer_service import ResmokeSymbolizer
 
 
 # pylint: disable=attribute-defined-outside-init
@@ -54,7 +55,9 @@ def combine(cls, *reports):
 
         for report in reports:
             if not isinstance(report, TestReport):
-                raise TypeError("reports must be a list of TestReport instances")
+                raise TypeError(
+                    f"reports must be a list of TestReport instances, current report is {type(report)}"
+                )
 
             with report._lock:  # pylint: disable=protected-access
                 for test_info in report.test_infos:
@@ -136,6 +139,15 @@ def startTest(self, test):  # pylint: disable=invalid-name
     def stopTest(self, test):  # pylint: disable=invalid-name
         """Call after 'test' has run."""
 
+        # check if there are stacktrace files, if so, invoke the symbolizer here.
+        # If there are no stacktrace files for this job, we do not need to invoke the symbolizer at all.
+        # Take a lock to download the debug symbols if it hasn't already been downloaded.
+        # log symbolized output to test.logger.info()
+
+        symbolizer = ResmokeSymbolizer()
+        symbolizer.symbolize_test_logs(test)
+        # symbolization completed
+
         unittest.TestResult.stopTest(self, test)
 
         with self._lock: