Backed out 3 changesets (bug 1713815) for causing perfdoc failures. C…

…LOSED TREE

Backed out changeset f57299491e76 (bug 1713815)
Backed out changeset a6bbbb4b43f1 (bug 1713815)
Backed out changeset 5672d15a0b2c (bug 1713815)

taskcluster/ci/test/browsertime-desktop.yml

@@ -122,53 +122,6 @@ browsertime-tp6:
                     firefox: 2
                     default: 3
 
-browsertime-responsiveness:
-    <<: *tp6-defaults
-    raptor-subtests:
-        - cnn-nav
-    mozharness:
-        extra-options:
-            by-test-platform:
-                windows10-.*:
-                    - --cold
-                    - --browsertime
-                    - --browsertime-no-ffwindowrecorder
-                    - --conditioned-profile=settled
-                default:
-                    - --cold
-                    - --browsertime
-                    - --conditioned-profile=settled
-    run-on-projects:
-        by-variant:
-            fission:
-                by-test-platform:
-                    windows.*-32.*: []
-                    .*clang.*: []
-                    default: [mozilla-central]
-            default:
-                by-app:
-                    firefox:
-                        by-test-platform:
-                            .*clang.*: []
-                            linux.*shippable.*: [mozilla-central]
-                            macosx1015.*shippable.*: [mozilla-central]
-                            windows10-64.*shippable.*: [mozilla-central]
-                            default: []
-                    default: []
-    variants:
-        by-app:
-            firefox:
-                by-test-platform:
-                    .*clang.*: []
-                    windows10-64.*shippable-qr/.*: [fission]
-                    (?!windows10-64).*shippable-qr/.*: [fission]
-                    default: []
-            default: []
-    tier:
-        by-app:
-            firefox: 2
-            default: 3
-
 browsertime-tp6-essential:
     <<: *tp6-defaults
     raptor-subtests:

taskcluster/ci/test/test-sets.yml

@@ -90,7 +90,6 @@ browsertime:
     - browsertime-youtube-playback
     - browsertime-custom
     - browsertime-first-install
-    - browsertime-responsiveness
 
 browsertime-profiling:
     - browsertime-tp6-profiling

taskcluster/docker/visual-metrics/run-visual-metrics.py

@@ -320,12 +320,9 @@ def main(log, args):
         for site in browsertime_json:
             for video in site["files"]["video"]:
                 count += 1
-                name = job["test_name"]
-                if "alias" in site["info"] and site["info"]["alias"].strip() != "":
-                    name = "%s.%s" % (name, site["info"]["alias"])
                 jobs.append(
                     Job(
-                        test_name=name,
+                        test_name=job["test_name"],
                         extra_options=len(job["extra_options"]) > 0
                         and job["extra_options"]
                         or jobs_json["extra_options"],

testing/mozbase/manifestparser/manifestparser/util.py

@@ -4,7 +4,6 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-import ast
 import os
 
 
@@ -26,23 +25,3 @@ def normsep(path):
         else:
             path = path.replace(os.altsep, "/")
     return path
-
-
-def evaluate_list_from_string(list_string):
-    """
-    This is a utility function for converting a string obtained from a manifest
-    into a list. If the string is not a valid list when converted, an error will be
-    raised from `ast.eval_literal`. For example, you can convert entries like this
-    into a list:
-    ```
-        test_settings=
-            ["hello", "world"],
-            [1, 10, 100],
-        values=
-            5,
-            6,
-            7,
-            8,
-    ```
-    """
-    return ast.literal_eval("[" + "".join(list_string.strip(",").split("\n")) + "]")

testing/mozbase/manifestparser/tests/manifest.ini

@@ -10,4 +10,3 @@ subsuite = mozbase
 [test_convert_symlinks.py]
 disabled = https://bugzilla.mozilla.org/show_bug.cgi?id=920938
 [test_default_overrides.py]
-[test_util.py]

testing/perfdocs/generated/raptor.rst

@@ -4217,151 +4217,6 @@ Tests for page-load performance. The links direct to the actual websites that ar
 
 
 
-Interactive
------------
-Browsertime tests that interact with the webpage. Includes responsiveness tests as they make use of this support for navigation. These form of tests allow the specification of browsertime commands through the test manifest.
-
-.. dropdown:: cnn-nav (Navigates to cnn main page, then to the world sub-page.)
-   :container: + anchor-id-cnn-nav-i
-
-   * **accept zero vismet**: true
-   * **alert on**: fcp, loadtime
-   * **alert threshold**: 2.0
-   * **apps**: firefox, chrome, chromium
-   * **browser cycles**: 10
-   * **expected**: pass
-   * **gecko profile entries**: 14000000
-   * **gecko profile interval**: 1
-   * **interactive**: true
-   * **lower is better**: true
-   * **measure**: fnbpaint, fcp, dcf, loadtime
-   * **page cycles**: 25
-   * **page timeout**: 60000
-   * **playback**: mitmproxy
-   * **playback pageset manifest**: mitm6-windows-firefox-cnn-nav.manifest
-   * **playback version**: 6.0.2
-   * **test cmds**:  ["measure.start", "landing"], ["navigate", "https://www.cnn.com"], ["wait.byTime", 4000], ["measure.stop", ""], ["measure.start", "world"], ["click.byXpathAndWait", "/html/body/div[5]/div/div/header/div/div[1]/div/div[2]/nav/ul/li[2]/a"], ["wait.byTime", 1000], ["measure.stop", ""],
-   * **test url**: `<https://www.cnn.com/>`__
-   * **type**: pageload
-   * **unit**: ms
-   * **use live sites**: false
-   * **Test Task**:
-      * test-linux1804-64-clang-trunk-qr/opt
-            * browsertime-responsiveness-firefox-cnn-nav-e10s: None
-      * test-linux1804-64-qr/opt
-            * browsertime-responsiveness-firefox-cnn-nav-e10s: None
-      * test-linux1804-64-shippable-qr/opt
-            * browsertime-responsiveness-chrome-cnn-nav-e10s: None
-            * browsertime-responsiveness-chromium-cnn-nav-e10s: None
-            * browsertime-responsiveness-firefox-cnn-nav-e10s: mozilla-central
-            * browsertime-responsiveness-firefox-cnn-nav-fis-e10s: mozilla-central
-      * test-macosx1014-64-shippable-qr/opt
-            * browsertime-responsiveness-chrome-cnn-nav-e10s: None
-            * browsertime-responsiveness-chromium-cnn-nav-e10s: None
-            * browsertime-responsiveness-firefox-cnn-nav-e10s: None
-            * browsertime-responsiveness-firefox-cnn-nav-fis-e10s: mozilla-central
-      * test-macosx1015-64-shippable-qr/opt
-            * browsertime-responsiveness-chrome-cnn-nav-e10s: None
-            * browsertime-responsiveness-chromium-cnn-nav-e10s: None
-            * browsertime-responsiveness-firefox-cnn-nav-e10s: mozilla-central
-            * browsertime-responsiveness-firefox-cnn-nav-fis-e10s: mozilla-central
-      * test-windows10-32-qr/opt
-            * browsertime-responsiveness-firefox-cnn-nav-e10s: None
-      * test-windows10-32-shippable-qr/opt
-            * browsertime-responsiveness-chrome-cnn-nav-e10s: None
-            * browsertime-responsiveness-chromium-cnn-nav-e10s: None
-            * browsertime-responsiveness-firefox-cnn-nav-e10s: None
-            * browsertime-responsiveness-firefox-cnn-nav-fis-e10s: None
-      * test-windows10-64-qr/opt
-            * browsertime-responsiveness-firefox-cnn-nav-e10s: None
-      * test-windows10-64-ref-hw-2017-qr/opt
-            * browsertime-responsiveness-firefox-cnn-nav-e10s: None
-      * test-windows10-64-shippable-qr/opt
-            * browsertime-responsiveness-chrome-cnn-nav-e10s: None
-            * browsertime-responsiveness-chromium-cnn-nav-e10s: None
-            * browsertime-responsiveness-firefox-cnn-nav-e10s: mozilla-central
-            * browsertime-responsiveness-firefox-cnn-nav-fis-e10s: mozilla-central
-
-
-.. dropdown:: facebook-nav (Navigates to facebook, then the sub-pages friends, marketplace, groups.)
-   :container: + anchor-id-facebook-nav-i
-
-   * **accept zero vismet**: true
-   * **alert on**: fcp, loadtime
-   * **alert threshold**: 2.0
-   * **apps**: firefox, chrome, chromium
-   * **browser cycles**: 25
-   * **expected**: pass
-   * **gecko profile entries**: 14000000
-   * **gecko profile interval**: 1
-   * **interactive**: true
-   * **lower is better**: true
-   * **measure**: fnbpaint, fcp, dcf, loadtime
-   * **page cycles**: 25
-   * **page timeout**: 90000
-   * **playback**: mitmproxy
-   * **playback pageset manifest**: mitm6-windows-firefox-facebook-nav.manifest
-   * **playback version**: 6.0.2
-   * **test cmds**:  ["measure.start", "landing"], ["navigate", "https://www.facebook.com/"], ["wait.byTime", 5000], ["measure.stop", ""], ["measure.start", "friends"], ["navigate", "https://www.facebook.com/friends/"], ["wait.byTime", 5000], ["measure.stop", ""], ["measure.start", "marketplace"], ["navigate", "https://www.facebook.com/marketplace"], ["wait.byTime", 5000], ["measure.stop", ""], ["measure.start", "groups"], ["navigate", "https://www.facebook.com/groups/discover/"], ["wait.byTime", 5000], ["measure.stop", ""],
-   * **test url**: `<https://www.facebook.com>`__
-   * **type**: pageload
-   * **unit**: ms
-   * **use live sites**: false
-
-
-.. dropdown:: reddit-billgates-ama (Navigates from the Bill Gates AMA to the Reddit members section.)
-   :container: + anchor-id-reddit-billgates-ama-i
-
-   * **accept zero vismet**: true
-   * **alert on**: fcp, loadtime
-   * **alert threshold**: 2.0
-   * **apps**: firefox, chrome, chromium
-   * **browser cycles**: 25
-   * **expected**: pass
-   * **gecko profile entries**: 14000000
-   * **gecko profile interval**: 1
-   * **interactive**: true
-   * **lower is better**: true
-   * **measure**: fnbpaint, fcp, dcf, loadtime
-   * **page cycles**: 25
-   * **page timeout**: 60000
-   * **playback**: mitmproxy
-   * **playback pageset manifest**: mitm6-windows-firefox-reddit-billgates-ama.manifest
-   * **playback version**: 6.0.2
-   * **test cmds**:  ["measure.start", "billg-ama"], ["navigate", "https://www.reddit.com/r/IAmA/comments/m8n4vt/im_bill_gates_cochair_of_the_bill_and_melinda/"], ["wait.byTime", 5000], ["measure.stop", ""], ["measure.start", "members"], ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div[3]/div[2]/div/div[1]/div/div[4]/div[1]/div"], ["wait.byTime", 1000], ["measure.stop", ""],
-   * **test url**: `<https://www.reddit.com/>`__
-   * **type**: pageload
-   * **unit**: ms
-   * **use live sites**: false
-
-
-.. dropdown:: reddit-billgates-post (Navigates the `thisisbillgates` user starting at the main user page, then to the posts, comments, hot, and top sections.)
-   :container: + anchor-id-reddit-billgates-post-i
-
-   * **accept zero vismet**: true
-   * **alert on**: fcp, loadtime
-   * **alert threshold**: 2.0
-   * **apps**: firefox, chrome, chromium
-   * **browser cycles**: 25
-   * **expected**: pass
-   * **gecko profile entries**: 14000000
-   * **gecko profile interval**: 1
-   * **interactive**: true
-   * **lower is better**: true
-   * **measure**: fnbpaint, fcp, dcf, loadtime
-   * **page cycles**: 25
-   * **page timeout**: 90000
-   * **playback**: mitmproxy
-   * **playback pageset manifest**: mitm6-windows-firefox-reddit-billgates-post.manifest
-   * **playback version**: 6.0.2
-   * **test cmds**:  ["measure.start", "billg"], ["navigate", "https://www.reddit.com/user/thisisbillgates/"], ["wait.byTime", 5000], ["measure.stop", ""], ["measure.start", "posts"], ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[2]"], ["wait.byTime", 15000], ["measure.stop", ""], ["measure.start", "comments"], ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[3]"], ["wait.byTime", 15000], ["measure.stop", ""], ["measure.start", "hot"], ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[2]"], ["wait.byTime", 15000], ["measure.stop", ""], ["measure.start", "top"], ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[3]"], ["wait.byTime", 15000], ["measure.stop", ""],
-   * **test url**: `<https://www.reddit.com/user/thisisbillgates/>`__
-   * **type**: pageload
-   * **unit**: ms
-   * **use live sites**: false
-
-
-
 Live
 ----
 A set of test pages that are run as live sites instead of recorded versions. These tests are available on all browsers, on all platforms. (WX: WebExtension, BT: Browsertime, FF: Firefox, CH: Chrome, CU: Chromium, GV: Geckoview, RB: Refbrow, FE: Fenix, CH-M: Chrome mobile)

testing/performance/hooks_recording.py

@@ -7,8 +7,6 @@
 import platform
 from pathlib import Path
 
-from manifestparser.util import evaluate_list_from_string
-
 from mozperftest.test.browsertime import add_options, add_option
 
 options = [
@@ -82,9 +80,4 @@ def before_runs(env):
         add_option(env, "browsertime.screenshot", "true")
         add_option(env, "browsertime.testName", test_site.get("name"))
 
-        # pack array into string for transport to javascript, is there a better way?
-        cmds = evaluate_list_from_string(test_site.get("test_cmds", "[]"))
-        parsed_cmds = [":::".join([str(i) for i in item]) for item in cmds if item]
-        add_option(env, "browsertime.commands", ";;;".join(parsed_cmds))
-
         print("Recording %s to file: %s" % (test_site.get("url"), recording_file))