Bug 1662706 - add a fuzzy runner r=sparky,necko-reviewers,rstewart DO…

…NTBUILD

Differential Revision: https://phabricator.services.mozilla.com/D89123

browser/base/content/test/perftest.ini

@@ -0,0 +1 @@
+[perftest_browser_xhtml_dom.js]

browser/base/moz.build

@@ -62,6 +62,10 @@ BROWSER_CHROME_MANIFESTS += [
     'content/test/zoom/browser.ini',
 ]
 
+PERFTESTS_MANIFESTS += [
+    'content/test/perftest.ini'
+]
+
 DEFINES['MOZ_APP_VERSION'] = CONFIG['MOZ_APP_VERSION']
 DEFINES['MOZ_APP_VERSION_DISPLAY'] = CONFIG['MOZ_APP_VERSION_DISPLAY']
 

moz.build

@@ -149,7 +149,7 @@ DIRS += [
     'build',
     'config',
     'python',
-    'testing/mozbase',
+    'testing',
     'third_party/python',
 ]
 

netwerk/test/moz.build

@@ -14,6 +14,11 @@ XPCSHELL_TESTS_MANIFESTS += [
     'unit_ipc/xpcshell.ini',
 ]
 
+PERFTESTS_MANIFESTS += [
+  'perf/perftest.ini',
+  'unit/perftest.ini'
+]
+
 if CONFIG['FUZZING_INTERFACES']:
     TEST_DIRS += [
         'fuzz'

netwerk/test/perf/perftest.ini

@@ -0,0 +1,7 @@
+[perftest_http3_cloudflareblog.js]
+[perftest_http3_facebook_scroll.js]
+[perftest_http3_google_image.js]
+[perftest_http3_google_search.js]
+[perftest_http3_lucasquicfetch.js]
+[perftest_http3_youtube_watch.js]
+[perftest_http3_youtube_watch_scroll.js]

netwerk/test/unit/perftest.ini

@@ -0,0 +1 @@
+[test_http3_perf.js]

python/mozbuild/mozbuild/frontend/context.py

@@ -1998,6 +1998,10 @@ def aggregate(files):
                                   """List of manifest files defining python unit tests.
         """),
 
+    'PERFTESTS_MANIFESTS': (ManifestparserManifestList, list,
+                            """List of manifest files defining MozPerftest performance tests.
+        """),
+
     'CRAMTEST_MANIFESTS': (ManifestparserManifestList, list,
                            """List of manifest files defining cram unit tests.
         """),

python/mozbuild/mozbuild/testing.py

@@ -58,6 +58,7 @@
     MOCHITEST_CHROME=('chrome', 'testing/mochitest', 'chrome', True),
     WEBRTC_SIGNALLING_TEST=('steeplechase', 'steeplechase', '.', True),
     XPCSHELL_TESTS=('xpcshell', 'xpcshell', '.', True),
+    PERFTESTS=('perftest', 'testing/perf', 'perf', True)
 )
 
 # reftests, wpt, and puppeteer all have their own manifest formats

python/mozperftest/mozperftest/fzf/__init__.py

@@ -0,0 +1,3 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.

python/mozperftest/mozperftest/fzf/fzf.py

@@ -0,0 +1,108 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import os
+import sys
+import subprocess
+from pathlib import Path
+import json
+
+from mozterm import Terminal
+from mozboot.util import get_state_dir
+from mozbuild.util import ensure_subprocess_env
+from distutils.spawn import find_executable
+
+
+HERE = Path(__file__).parent.resolve()
+SRC_ROOT = (HERE / ".." / ".." / ".." / "..").resolve()
+PREVIEW_SCRIPT = HERE / "preview.py"
+FZF_HEADER = """
+Please select a performance test to execute.
+{shortcuts}
+""".strip()
+
+fzf_shortcuts = {
+    "ctrl-t": "toggle-all",
+    "alt-bspace": "beginning-of-line+kill-line",
+    "?": "toggle-preview",
+}
+
+fzf_header_shortcuts = [
+    ("select", "tab"),
+    ("accept", "enter"),
+    ("cancel", "ctrl-c"),
+    ("cursor-up", "up"),
+    ("cursor-down", "down"),
+]
+
+
+def run_fzf(cmd, tasks):
+    env = dict(os.environ)
+    env.update(
+        {"PYTHONPATH": os.pathsep.join([p for p in sys.path if "requests" in p])}
+    )
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stdin=subprocess.PIPE,
+        env=ensure_subprocess_env(env),
+        universal_newlines=True,
+    )
+    out = proc.communicate("\n".join(tasks))[0].splitlines()
+    selected = []
+    query = None
+    if out:
+        query = out[0]
+        selected = out[1:]
+    return query, selected
+
+
+def format_header():
+    terminal = Terminal()
+    shortcuts = []
+    for action, key in fzf_header_shortcuts:
+        shortcuts.append(
+            "{t.white}{action}{t.normal}: {t.yellow}<{key}>{t.normal}".format(
+                t=terminal, action=action, key=key
+            )
+        )
+    return FZF_HEADER.format(shortcuts=", ".join(shortcuts), t=terminal)
+
+
+def autodetect(path):
+    from mozperftest.script import ScriptInfo
+
+    return ScriptInfo.detect_type(path).name
+
+
+def select(test_objects):
+    mozbuild_dir = Path(Path.home(), ".mozbuild")
+    os.makedirs(str(mozbuild_dir), exist_ok=True)
+    cache_file = Path(mozbuild_dir, ".perftestfuzzy")
+
+    with cache_file.open("w") as f:
+        f.write(json.dumps(test_objects))
+
+    def _display(task):
+        flavor = autodetect(task["path"])
+        path = task["path"].replace(str(SRC_ROOT), "")
+        return f"[{flavor}] {path}"
+
+    candidate_tasks = [_display(t) for t in test_objects]
+    fzf_bin = find_executable("fzf", str(Path(get_state_dir(), "fzf", "bin")))
+    key_shortcuts = [k + ":" + v for k, v in fzf_shortcuts.items()]
+
+    base_cmd = [
+        fzf_bin,
+        "-m",
+        "--bind",
+        ",".join(key_shortcuts),
+        "--header",
+        format_header(),
+        "--preview-window=right:50%",
+        "--print-query",
+        "--preview",
+        sys.executable + ' {} -t "{{+f}}"'.format(str(PREVIEW_SCRIPT)),
+    ]
+    query_str, tasks = run_fzf(base_cmd, sorted(candidate_tasks))
+    return tasks

python/mozperftest/mozperftest/fzf/preview.py

@@ -0,0 +1,88 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+This file is executed by fzf through the command line and needs to
+work in a standalone way on any Python 3 environment.
+
+This is why it alters PATH,making the assumption it's executed
+from within a source tree. Do not add dependencies unless they
+are in the source tree and added in SEARCH_PATHS.
+"""
+import argparse
+import sys
+import json
+from pathlib import Path
+import importlib.util
+
+
+HERE = Path(__file__).parent.resolve()
+SRC_ROOT = (HERE / ".." / ".." / ".." / "..").resolve()
+# make sure esprima is in the path
+SEARCH_PATHS = [
+    ("third_party", "python", "esprima"),
+]
+
+for path in SEARCH_PATHS:
+    path = Path(SRC_ROOT, *path)
+    if path.exists():
+        sys.path.insert(0, str(path))
+
+
+def get_test_objects():
+    """Loads .perftestfuzzy and returns its content.
+
+    The cache file is produced by the main fzf script and is used
+    as a way to let the preview script grab test_objects from the
+    mach command
+    """
+    cache_file = Path(Path.home(), ".mozbuild", ".perftestfuzzy")
+    with cache_file.open() as f:
+        return json.loads(f.read())
+
+
+def plain_display(taskfile):
+    """Preview window display.
+
+    Returns the reST summary for the perf test script.
+    """
+    # Lame way to catch the ScriptInfo class without loading mozperftest
+    script_info = HERE / ".." / "script.py"
+    spec = importlib.util.spec_from_file_location(
+        name="script.py", location=str(script_info)
+    )
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    ScriptInfo = module.ScriptInfo
+
+    with open(taskfile) as f:
+        tasklist = [line.strip() for line in f]
+    script_name = tasklist[0].split(" ")[-1]
+    for ob in get_test_objects():
+        if ob["path"].endswith(script_name):
+            print(ScriptInfo(ob["path"]))
+            return
+
+
+def process_args(args):
+    """Process preview arguments."""
+    argparser = argparse.ArgumentParser()
+    argparser.add_argument(
+        "-t",
+        "--tasklist",
+        type=str,
+        default=None,
+        help="Path to temporary file containing the selected tasks",
+    )
+    return argparser.parse_args(args=args)
+
+
+def main(args=None):
+    if args is None:
+        args = sys.argv[1:]
+    args = process_args(args)
+    plain_display(args.tasklist)
+
+
+if __name__ == "__main__":
+    main()

python/mozperftest/mozperftest/mach_commands.py

@@ -31,9 +31,10 @@ class Perftest(MachCommandBase):
         parser=get_perftest_parser,
     )
     def run_perftest(self, **kwargs):
+        from pathlib import Path
+
         push_to_try = kwargs.pop("push_to_try", False)
         if push_to_try:
-            from pathlib import Path
 
             sys.path.append(str(Path(self.topsrcdir, "tools", "tryselect")))
 
@@ -66,13 +67,53 @@ def run_perftest(self, **kwargs):
             push_to_try("perftest", "perftest", try_task_config=task_config)
             return
 
+        # user selection with fuzzy UI
+        from mozperftest.utils import ON_TRY
+
+        if not ON_TRY and kwargs.get("tests", []) == []:
+            from moztest.resolve import TestResolver
+            from mozperftest.fzf.fzf import select
+            from mozperftest.script import ScriptInfo, ScriptType
+
+            resolver = self._spawn(TestResolver)
+            test_objects = list(resolver.resolve_tests(paths=None, flavor="perftest"))
+
+            def full_path(s):
+                relative = s.split()[-1].lstrip(os.sep)
+                return str(Path(self.topsrcdir, relative))
+
+            kwargs["tests"] = [full_path(s) for s in select(test_objects)]
+
+            if kwargs["tests"] == []:
+                print("\nNo selection. Bye!")
+                return
+
+            if len(kwargs["tests"]) > 1:
+                print("\nSorry no support yet for multiple local perftest")
+                return
+
+            sel = "\n".join(kwargs["tests"])
+            print("\nGood job! Best selection.\n%s" % sel)
+
+            # if the script is xpcshell, we can force the flavor here
+            script_type = ScriptInfo.detect_type(kwargs["tests"][0])
+            if script_type == ScriptType.xpcshell:
+                kwargs["flavor"] = script_type.name
+            else:
+                # we set the value only if not provided (so "mobile-browser"
+                # can be picked)
+                if "flavor" not in kwargs:
+                    kwargs["flavor"] = "desktop-browser"
+
         # run locally
         MachCommandBase.activate_virtualenv(self)
 
         from mozperftest.runner import run_tests
 
         run_tests(mach_cmd=self, **kwargs)
 
+        print("\nFirefox. Fast For Good.\n")
+
 
 @CommandProvider
 class PerftestTests(MachCommandBase):