Bug 1883123: fully ignore changes in UpdateSettings and ChannelPrefs …

…frameworks for update verify tests r=jcristau

These new frameworks unavoidably change in unpredictable ways with each build (signing them generate CodeResources, and modifies the included binary). Because of this we have no choice but to simply ignore differences in them when running update verify.

This generally should be OK, because the happy path of updates is one where the on-disk version of these files already exist, and would not be altered as part of updating. The main risk comes in around the first update to include them, where they _will_ be added to existing installs. Ideally we would probably want additional tests around this that remove these frameworks before applying the updates, which we would _not_ want to ignore these files for -- but I doubt we can get that ready in time for 125 hitting Beta.

Differential Revision: https://phabricator.services.mozilla.com/D203801

tools/update-verify/release/common/check_updates.sh

@@ -110,6 +110,21 @@ check_updates () {
     ignore_coderesources=
   fi
 
+  # On Mac, there are two Frameworks that are not included with updates, and
+  # which change with every build. Because of this, we ignore differences in
+  # them in `compare-directories.py`. The best verification we can do for them
+  # is that they still exist.
+  if [[ $update_platform == Darwin_* ]]; then
+    if ! compgen -G "source/${platform_dirname}/Contents/MacOS/updater.app/Contents/Frameworks/UpdateSettings.framework" >/dev/null; then
+      echo "TEST-UNEXPECTED-FAIL: UpdateSettings.framework doesn't exist after update"
+      return 4
+    fi
+    if ! compgen -G "source/${platform_dirname}/Contents/Frameworks/ChannelPrefs.framework" >/dev/null; then
+      echo "TEST-UNEXPECTED-FAIL: ChannelPrefs.framework doesn't exist after update"
+      return 5
+    fi
+  fi
+
   ../compare-directories.py source/${platform_dirname} target/${platform_dirname} ${channel} ${ignore_coderesources} > "${diff_file}"
   diffErr=$?
   cat "${diff_file}"

tools/update-verify/release/compare-directories.py

@@ -29,7 +29,6 @@
         # this can be removed once each channel has a watershed above 59.0b2 (from bug 1431342)
         "files": [
             "defaults/pref/channel-prefs.js",
-            "Contents/Resources/defaults/pref/channel-prefs.js",
         ],
         "channel_prefix": ["aurora", "beta", "release", "esr"],
         "side": "source",
@@ -39,7 +38,6 @@
         # updates from a beta to an RC build, the latter specifies the release channel
         "files": [
             "defaults/pref/channel-prefs.js",
-            "Contents/Resources/defaults/pref/channel-prefs.js",
         ],
         "channel_prefix": ["beta"],
         "side": "target",
@@ -52,7 +50,6 @@
         # updates from an RC to a beta build
         "files": [
             "defaults/pref/channel-prefs.js",
-            "Contents/Resources/defaults/pref/channel-prefs.js",
         ],
         "channel_prefix": ["beta"],
         "side": "source",
@@ -69,7 +66,6 @@
         # to break before applying this transform.
         "files": [
             "defaults/pref/channel-prefs.js",
-            "Contents/Resources/defaults/pref/channel-prefs.js",
         ],
         "channel_prefix": ["aurora", "beta", "release", "esr"],
         "side": "target",
@@ -80,28 +76,28 @@
         # updates from a beta to an RC build, the latter specifies the release channel
         # on mac, we actually have both files. The second location is the real
         # one but we copy to the first to run the linux64 updater
-        "files": ["update-settings.ini", "Contents/Resources/update-settings.ini"],
+        "files": ["update-settings.ini"],
         "channel_prefix": ["beta"],
         "side": "target",
         "substitution": [
             "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-release\n",
             "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-beta,firefox-mozilla-release\n",
         ],
     },
-    {
-        # updates from an RC to a beta build
-        # on mac, we only need to modify the legit file this time. unpack_build
-        # handles the copy for the updater in both source and target
-        "files": ["Contents/Resources/update-settings.ini"],
-        "channel_prefix": ["beta"],
-        "side": "source",
-        "substitution": [
-            "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-release\n",
-            "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-beta,firefox-mozilla-release\n",
-        ],
-    },
 ]
 
+# Files that are expected to be different, but cannot be transformed to get a useful diff.
+# This should generally only be used for files that have unpredictable contents, eg:
+# things that are signed but not updated.
+IGNORE_FILES = (
+    "Contents/MacOS/updater.app/Contents/Frameworks/UpdateSettings.framework/Resources/Info.plist",
+    "Contents/MacOS/updater.app/Contents/Frameworks/UpdateSettings.framework/_CodeSignature/CodeResources",
+    "Contents/MacOS/updater.app/Contents/Frameworks/UpdateSettings.framework/UpdateSettings",
+    "Contents/Frameworks/ChannelPrefs.framework/Resources/Info.plist",
+    "Contents/Frameworks/ChannelPrefs.framework/_CodeSignature/CodeResources",
+    "Contents/Frameworks/ChannelPrefs.framework/ChannelPrefs",
+)
+
 
 def walk_dir(path):
     all_files = []
@@ -170,6 +166,14 @@ def compare_common_files(files, channel, source_dir, target_dir):
             source_file
         ) != hash_file(target_file):
             logging.info("Difference found in {}".format(filename))
+            if filename in IGNORE_FILES:
+                logging.info(
+                    "Ignoring difference in {} because it is listed in IGNORE_FILES".format(
+                        filename
+                    )
+                )
+                continue
+
             file_contents = {
                 "source": open(source_file).readlines(),
                 "target": open(target_file).readlines(),