SERVER-67058 Create libdeps metrics

erikbeebe · Aug 9, 2022 · fc3674c · fc3674c
1 parent 38d4d52
commit fc3674c
Show file tree

Hide file tree

Showing 10 changed files with 128 additions and 48 deletions.
diff --git a/SConstruct b/SConstruct
@@ -2259,7 +2259,7 @@ if env['_LIBDEPS'] == '$_LIBDEPS_OBJS':
     env["BUILDERS"]["StaticLibrary"].action = SCons.Action.Action(
         write_uuid_to_file, "Generating placeholder library $TARGET")
 
-import libdeps
+import libdeps_tool as libdeps
 
 libdeps.setup_environment(
     env,

diff --git a/buildscripts/libdeps/libdeps/analyzer.py b/buildscripts/libdeps/libdeps/analyzer.py
@@ -368,14 +368,19 @@ def counter_factory(dependency_graph, counters, progressbar=True):
         counters = [counters]
 
     counter_objs = []
-    for counter in counters:
-        if counter in counter_map:
+    if CountTypes.ALL.name in counters:
+        for counter in counter_map:
             counter_obj = counter_map[counter](dependency_graph)
             counter_obj.set_progress(progressbar)
             counter_objs.append(counter_obj)
-
-        else:
-            print(f"Skipping unknown counter: {counter}")
+    else:
+        for counter in counters:
+            if counter in counter_map:
+                counter_obj = counter_map[counter](dependency_graph)
+                counter_obj.set_progress(progressbar)
+                counter_objs.append(counter_obj)
+            else:
+                print(f"Skipping unknown counter: {counter}")
 
     return counter_objs
 

diff --git a/etc/pip/toolchain-requirements.txt b/etc/pip/toolchain-requirements.txt
@@ -16,3 +16,4 @@
 -r components/platform.req
 
 -r components/build_metrics.req
+-r components/libdeps.req
diff --git a/site_scons/libdeps.py → site_scons/libdeps_tool.py b/site_scons/libdeps.py → site_scons/libdeps_tool.py
diff --git a/site_scons/site_tools/build_metrics/__init__.py b/site_scons/site_tools/build_metrics/__init__.py
@@ -37,6 +37,7 @@
 from .artifacts import CollectArtifacts
 from .scons import SConsStats
 from .cache_dir import CacheDirCollector, CacheDirValidateWithMetrics
+from .libdeps import LibdepsCollector
 
 _SEC_TO_NANOSEC_FACTOR = 1000000000.0
 _METRICS_COLLECTORS = []
@@ -48,6 +49,7 @@ def finalize_build_metrics(env):
     for m in _METRICS_COLLECTORS:
         start_time = timer()
         sys.stdout.write(f"Processing {m.get_name()}...")
+        sys.stdout.flush()
         key, value = m.finalize()
         sys.stdout.write(f" {timer() - start_time}s\n")
         metrics[key] = value
@@ -90,7 +92,8 @@ def generate(env, **kwargs):
         PerActionMetrics(),
         CollectArtifacts(env),
         SConsStats(),
-        CacheDirCollector()
+        CacheDirCollector(),
+        LibdepsCollector(env)
     ]
 
     env['CACHEDIR_CLASS'] = CacheDirValidateWithMetrics

diff --git a/site_scons/site_tools/build_metrics/artifacts.py b/site_scons/site_tools/build_metrics/artifacts.py
@@ -158,7 +158,7 @@ def __init__(self, env):
         self._env = env
         self._build_dir = env.get("BUILD_METRICS_ARTIFACTS_DIR", env.Dir('#').abspath)
         self._artifacts = []
-        self._bloaty_bin = env.get("BUILD_METRICS_BLOATY", "bloaty")
+        self._bloaty_bin = env.get("BUILD_METRICS_BLOATY", env.WhereIs('bloaty'))
         self._metrics = {"total_artifact_size": 0, "num_artifacts": 0, "artifacts": []}
 
     def get_name(self):
@@ -202,7 +202,7 @@ def _type_from_builder(builder) -> ArtifactType:
 
                 if type_ == ArtifactType.UNKNOWN and any(s in magic_out for s in _TEXT_IDENTIFIERS):
                     type_ = ArtifactType.TEXT
-            except puremagic.main.PureError:
+            except (puremagic.main.PureError, ValueError):
                 # exception means that puremagic failed to id the filetype. We'll
                 # fallback to file extension in this case.
                 pass

diff --git a/site_scons/site_tools/build_metrics/build_metrics_format.schema b/site_scons/site_tools/build_metrics/build_metrics_format.schema
@@ -106,10 +106,10 @@
                         "items": { "type": "string"}
                     },
                     "action": {"type" : "string"}
-                },
-                "required": ["array_index", "start_time", "end_time", "cpu_time", "builder", "mem_usage", "inputs", "outputs", "action"],
-                "additionalProperties": false
-            }
+                }
+            },
+            "required": ["array_index", "start_time", "end_time", "cpu_time", "builder", "mem_usage", "inputs", "outputs", "action"],
+            "additionalProperties": false
         },
         "cache_metrics": {
             "type": "object",
@@ -134,49 +134,65 @@
             "required": ["cache_artifacts", "push_time", "pull_time", "cache_size"],
             "additionalProperties": false
         },
-        "scons_metrics": {
+        "libdeps_metrics": {
             "type": "object",
             "properties": {
-                "memory": {
+                "NODE": {"type": "integer"},
+                "EDGE": {"type": "integer"},
+                "DIR_EDGE": {"type": "integer"},
+                "TRANS_EDGE": {"type": "integer"},
+                "DIR_PUB_EDGE": {"type": "integer"},
+                "PUB_EDGE": {"type": "integer"},
+                "PRIV_EDGE": {"type": "integer"},
+                "IF_EDGE": {"type": "integer"},
+                "PROG": {"type": "integer"},
+                "LIB": {"type": "integer"}
+            },
+            "additionalProperties": false
+        }
+    },
+    "scons_metrics": {
+        "type": "object",
+        "properties": {
+            "memory": {
+                "type": "object",
+                "properties": {
+                    "pre_read": {"$ref": "#/$defs/bytes"},
+                    "post_read": {"$ref": "#/$defs/bytes"},
+                    "pre_build": {"$ref": "#/$defs/bytes"},
+                    "post_build": {"$ref": "#/$defs/bytes"}
+                },
+                "required": ["pre_read", "post_read", "pre_build", "post_build"],
+                "additionalProperties": false
+            },
+            "time": {
+                "type": "object",
+                "properties": {
+                    "total": {"type": "number"},
+                    "sconscript_exec": {"type": "number"},
+                    "scons_exec": {"type": "number"},
+                    "command_exec": {"type": "number"}
+                },
+                "required": ["total", "sconscript_exec", "scons_exec", "command_exec"],
+                "additionalProperties": false
+            },
+            "counts": {
+                "type": "array",
+                "items": {
                     "type": "object",
                     "properties": {
+                        "array_index": {"type": "integer"},
+                        "item_name": {"type": "string"},
                         "pre_read": {"$ref": "#/$defs/bytes"},
                         "post_read": {"$ref": "#/$defs/bytes"},
                         "pre_build": {"$ref": "#/$defs/bytes"},
                         "post_build": {"$ref": "#/$defs/bytes"}
                     },
-                    "required": ["pre_read", "post_read", "pre_build", "post_build"],
+                    "required": ["array_index", "item_name", "pre_read", "post_read", "pre_build", "post_build"],
                     "additionalProperties": false
-                },
-                "time": {
-                    "type": "object",
-                    "properties": {
-                        "total": {"type": "number"},
-                        "sconscript_exec": {"type": "number"},
-                        "scons_exec": {"type": "number"},
-                        "command_exec": {"type": "number"}
-                    },
-                    "required": ["total", "sconscript_exec", "scons_exec", "command_exec"],
-                    "additionalProperties": false
-                },
-                "counts": {
-                    "type": "array",
-                    "items": {
-                        "type": "object",
-                        "properties": {
-                            "array_index": {"type": "integer"},
-                            "item_name": {"type": "string"},
-                            "pre_read": {"$ref": "#/$defs/bytes"},
-                            "post_read": {"$ref": "#/$defs/bytes"},
-                            "pre_build": {"$ref": "#/$defs/bytes"},
-                            "post_build": {"$ref": "#/$defs/bytes"}
-                        },
-                        "required": ["array_index", "item_name", "pre_read", "post_read", "pre_build", "post_build"],
-                        "additionalProperties": false
-                    }
-                },
-                "additionalProperties": false
+                }
             }
-        }
+        },
+        "additionalProperties": false
     }
 }
diff --git a/site_scons/site_tools/build_metrics/libdeps.py b/site_scons/site_tools/build_metrics/libdeps.py
@@ -0,0 +1,55 @@
+import os
+import sys
+import json
+from .protocol import BuildMetricsCollector
+
+import networkx
+
+# libdeps analyzer does not assume the root build directory, so we need to add its own root to the path
+dir_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.insert(0, os.path.join(dir_path, "..", "..", "..", "buildscripts", "libdeps"))
+
+from buildscripts.libdeps.libdeps.analyzer import counter_factory, LibdepsGraphAnalysis, GaJsonPrinter
+from buildscripts.libdeps.libdeps.graph import LibdepsGraph, CountTypes
+
+_ALLOWED_KEYS = set([
+    "NODE", "EDGE", "DIR_EDGE", "TRANS_EDGE", "DIR_PUB_EDGE", "PUB_EDGE", "PRIV_EDGE", "IF_EDGE",
+    "PROG", "LIB"
+])
+
+
+class LibdepsCollector(BuildMetricsCollector):
+    def __init__(self, env):
+        self._env = env
+
+    def get_name(self):
+        return "LibdepsCollector"
+
+    @staticmethod
+    def _libdeps(graph_file):
+        libdeps_graph = LibdepsGraph(graph=networkx.read_graphml(graph_file))
+
+        if libdeps_graph.graph['graph_schema_version'] == 1:
+            libdeps_graph = networkx.reverse_view(libdeps_graph)
+
+        return GaJsonPrinter(
+            LibdepsGraphAnalysis(counter_factory(libdeps_graph, CountTypes.ALL.name))).get_json()
+
+    @staticmethod
+    def _finalize(libdeps_graph_file):
+        out = {}
+        for key, value in json.loads(LibdepsCollector._libdeps(libdeps_graph_file)).items():
+            if key in _ALLOWED_KEYS:
+                out[key] = value
+        return out
+
+    def finalize(self):
+        libdeps_graph_file = self._env.get('LIBDEPS_GRAPH_FILE')
+        out = {}
+        if libdeps_graph_file is not None and os.path.exists(libdeps_graph_file.path):
+            out = self._finalize(libdeps_graph_file.path)
+        else:
+            print(
+                f"WARNING: libdeps graph file '{libdeps_graph_file}' could not be found. Skipping libdeps metrics"
+            )
+        return "libdeps_metrics", out
diff --git a/src/mongo/embedded/mongo_embedded/SConscript b/src/mongo/embedded/mongo_embedded/SConscript
@@ -1,7 +1,7 @@
 # -*- mode: python; -*-
 
 from functools import partial
-import libdeps
+import libdeps_tool as libdeps
 
 Import("env")
 Import("get_option")

diff --git a/src/mongo/embedded/stitch_support/SConscript b/src/mongo/embedded/stitch_support/SConscript
@@ -1,7 +1,7 @@
 # -*- mode: python; -*-
 
 from functools import partial
-import libdeps
+import libdeps_tool as libdeps
 
 Import("env")
 Import("get_option")
Original file line number	Diff line number	Diff line change
Expand Up		@@ -16,3 +16,4 @@
		-r components/platform.req

		-r components/build_metrics.req
		-r components/libdeps.req