Bug 1459526 - Handle full-range video in Webrender. r=gw,lsalzman

+ Begin to add video tests to ensure we ratchet towards correctness.
+ Test rec709 x (yuv420p, yuv420p10, gbrp) x (tv, pc) x codecs.
+ Just mark fuzziness for now. Better would be e.g. 16_127_233 'bad
  references'.

Differential Revision: https://phabricator.services.mozilla.com/D115298

dom/media/ipc/RemoteImageHolder.cpp

@@ -72,6 +72,7 @@ already_AddRefed<Image> RemoteImageHolder::DeserializeImage(
   pData.mStereoMode = descriptor.stereoMode();
   pData.mColorDepth = descriptor.colorDepth();
   pData.mYUVColorSpace = descriptor.yUVColorSpace();
+  pData.mColorRange = descriptor.colorRange();
   pData.mYChannel = ImageDataSerializer::GetYChannel(buffer, descriptor);
   pData.mCbChannel = ImageDataSerializer::GetCbChannel(buffer, descriptor);
   pData.mCrChannel = ImageDataSerializer::GetCrChannel(buffer, descriptor);

dom/media/test/reftest/color_quads/reftest.list

@@ -0,0 +1,71 @@
+# Reference image generated via https://jdashg.github.io/misc/colors/color-quads-16-127-235.html
+# Test videos encoded via ../gen_combos.py --write color_quads/720p.png
+
+# We're sort of testing two things here:
+# 1. Does a av1.webm video into the actual values we expect?
+# 2. Do other similar videos decode the same was as av1.webm?
+# We have this split because while each platform/compositor has its own inaccuracies,
+# each platform/compositor will have the *same* inaccuracies regardless of video.
+# So, we just need to first check if e.g. av1.webm decodes to what we expect,
+# and then we have generally trivially compare other codecs/containers to that.
+
+
+# -
+# yuv420p
+
+fuzzy(16-50,5234-5621) fuzzy-if(swgl,32-38,1760-91746) fuzzy-if(!webrender,16-38,5234-94640) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm  ../reftest_img.html?src=color_quads/720p.png
+fuzzy-if(Android,254-255,273680-273807) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.vp9.webm  ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm
+== ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.mp4   ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm
+fuzzy(0-2,75-225) fuzzy-if(Android,254-255,273680-273807) fuzzy-if(!Android&&!webrender,1-2,75-94070) fuzzy-if(OSX&&webrender,32-32,187407-187407) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.h264.mp4  ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm
+fuzzy(0-1,0-75) fuzzy-if(Android,254-255,273680-273807) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.vp9.mp4   ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm
+
+skip-if(!webrender||Android) fuzzy(16-48,8349-8818) fuzzy-if(winWidget&&swgl,38-38,184080-184080) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm  ../reftest_img.html?src=color_quads/720p.png
+skip-if(!webrender||Android) fuzzy-if(Android,255-255,273726-273726) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.vp9.webm  ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm
+skip-if(!webrender||Android) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.mp4   ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm
+skip-if(!webrender||Android) skip-if(winWidget&&swgl) fuzzy-if(Android,255-255,273726-273726) fuzzy-if(OSX||winWidget,2-34,184281-187407) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.h264.mp4  ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm
+skip-if(!webrender||Android) fuzzy-if(Android,255-255,273726-273726) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.vp9.mp4   ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm
+
+# -
+# yuv420p10
+
+skip-if(!webrender||Android) fuzzy(33-49,2499-2579) fuzzy-if(swgl,34-52,181053-270528) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm  ../reftest_img.html?src=color_quads/720p.png
+skip-if(!webrender||Android) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.vp9.webm  ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm
+skip-if(!webrender||Android) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.mp4   ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm
+#[2] skip-if(!webrender||Android) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.h264.mp4  ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm
+skip-if(!webrender||Android) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.vp9.mp4   ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm
+
+skip-if(!webrender||Android) fuzzy(33-49,174932-175092) fuzzy-if(swgl&&!winWidget,36-52,11553-11555) fuzzy-if(swgl&&winWidget,40-40,187200-187200) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm  ../reftest_img.html?src=color_quads/720p.png
+skip-if(!webrender||Android) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.vp9.webm  ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm
+skip-if(!webrender||Android) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.mp4   ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm
+#[2] skip-if(!webrender||Android) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.h264.mp4  ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm
+skip-if(!webrender||Android) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.vp9.mp4  ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm
+
+# Android is really broken in a variety of ways for p10.
+#[2]: yuv420p10 broken in h264.mp4: https://bugzilla.mozilla.org/show_bug.cgi?id=1711812
+
+
+# -
+# gbrp
+# Note: tv-gbrp doesn't really make sense, and we should consider dropping it.
+# Specifically, we should probably do (gbrp, ...(tv,pc)x(yuv,yuv10)) instead of (tv,pc)x(gbrp,yuv,yuv10)
+# That said, we should probably test a couple combos, at least. (But then again, why not all!)
+# !webrender does not support gbr
+
+skip-if(!webrender) skip-if(winWidget&&swgl) fuzzy(0-1,0-3600) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.gbrp.av1.webm  ../reftest_img.html?src=color_quads/720p.png
+skip-if(!webrender) skip-if(winWidget&&swgl) fuzzy(0-1,0-7200) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.gbrp.av1.webm  ../reftest_img.html?src=color_quads/720p.png
+
+skip-if(!webrender) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.gbrp.av1.mp4  ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.gbrp.av1.webm
+skip-if(!webrender) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.gbrp.av1.mp4  ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.gbrp.av1.webm
+
+# Our h264.mp4 doesn't handle gbrp, but *also* doesn't error properly.
+skip-if(!webrender) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.gbrp.h264.mp4  ../reftest_video.html?src=timeout
+skip-if(!webrender) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.gbrp.h264.mp4  ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.gbrp.h264.mp4
+
+# Our vp9 support doesn't handle gbrp
+skip-if(!webrender||OSX||winWidget) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.gbrp.vp9.webm  ../reftest_video.html?src=timeout
+skip-if(!webrender||!OSX) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.gbrp.vp9.webm  ../reftest_video.html?src=none
+skip-if(!webrender||!winWidget) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.gbrp.vp9.webm  ../reftest_video.html?src=none
+
+skip-if(!webrender) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.gbrp.vp9.webm  ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.gbrp.vp9.webm
+skip-if(!webrender) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.gbrp.vp9.mp4  ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.gbrp.vp9.webm
+skip-if(!webrender) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.gbrp.vp9.mp4  ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.gbrp.vp9.webm

dom/media/test/reftest/gen_combos.py

@@ -0,0 +1,250 @@
+#!/usr/bin/env python3
+
+import concurrent.futures
+import pathlib
+import subprocess
+import sys
+
+ARGS = sys.argv
+SRC_PATH = pathlib.Path(ARGS.pop())
+DIR = SRC_PATH.parent
+
+
+# crossCombine([{a:false},{a:5}], [{},{b:5}])
+# [{a:false}, {a:true}, {a:false,b:5}, {a:true,b:5}]
+def cross_combine(*args):
+    args = list(args)
+
+    def cross_combine2(listA, listB):
+        listC = []
+        for a in listA:
+            for b in listB:
+                c = dict()
+                c.update(a)
+                c.update(b)
+                listC.append(c)
+        return listC
+
+    res = [dict()]
+    while True:
+        try:
+            next = args.pop(0)
+        except IndexError:
+            break
+        res = cross_combine2(res, next)
+    return res
+
+
+def keyed_combiner(key, vals):
+    res = []
+    for v in vals:
+        d = dict()
+        d[key] = v
+        res.append(d)
+    return res
+
+
+# -
+
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+
+# -
+
+OGG = []
+WEBM_CODECS = ["av1", "vp9"]
+
+if "--all" in ARGS:
+    OGG = cross_combine(
+        [{"ext": "ogg"}], keyed_combiner("vcodec", ["theora", "vp8", "vp9"])
+    )
+    WEBM_CODECS += ["vp8"]
+
+MP4 = cross_combine([{"ext": "mp4"}], keyed_combiner("vcodec", ["av1", "h264", "vp9"]))
+
+WEBM = cross_combine([{"ext": "webm"}], keyed_combiner("vcodec", WEBM_CODECS))
+
+# -
+
+FORMAT_LIST = set(
+    [
+        "yuv420p",
+        "yuv420p10",
+        # 'yuv420p12',
+        # 'yuv420p16be',
+        # 'yuv420p16le',
+        "gbrp",
+    ]
+)
+
+if "--all" in ARGS:
+    FORMAT_LIST |= set(
+        [
+            "yuv420p",
+            "yuv420p10",
+            "yuv420p12",
+            "yuv420p16be",
+            "yuv420p16le",
+            "yuv422p",
+            "yuv422p10",
+            "yuv422p12",
+            "yuv422p16be",
+            "yuv422p16le",
+            "yuv444p",
+            "yuv444p10",
+            "yuv444p12",
+            "yuv444p16be",
+            "yuv444p16le",
+            "yuv411p",
+            "yuv410p",
+            "yuyv422",
+            "uyvy422",
+            "rgb24",
+            "bgr24",
+            "rgb8",
+            "bgr8",
+            "rgb444be",
+            "rgb444le",
+            "bgr444be",
+            "bgr444le",
+            # 'nv12', # Encoding not different than yuv420p?
+            # 'nv21', # Encoding not different than yuv420p?
+            "gbrp",
+            "gbrp9be",
+            "gbrp9le",
+            "gbrp10be",
+            "gbrp10le",
+            "gbrp12be",
+            "gbrp12le",
+            "gbrp14be",
+            "gbrp14le",
+            "gbrp16be",
+            "gbrp16le",
+        ]
+    )
+
+FORMATS = keyed_combiner("format", list(FORMAT_LIST))
+
+RANGE = keyed_combiner("range", ["tv", "pc"])
+
+CSPACE_LIST = set(
+    [
+        "bt709",
+        # 'bt2020',
+    ]
+)
+
+if "--all" in ARGS:
+    CSPACE_LIST |= set(
+        [
+            "bt709",
+            "bt2020",
+            "bt601-6-525",  # aka smpte170m NTSC
+            "bt601-6-625",  # aka bt470bg PAL
+        ]
+    )
+CSPACE_LIST = list(CSPACE_LIST)
+
+# -
+
+COMBOS = cross_combine(
+    WEBM + MP4 + OGG,
+    FORMATS,
+    RANGE,
+    keyed_combiner("src_cspace", CSPACE_LIST),
+    keyed_combiner("dst_cspace", CSPACE_LIST),
+)
+
+# -
+
+print(f"{len(COMBOS)} combinations...")
+
+todo = []
+for c in COMBOS:
+    dst_name = ".".join(
+        [
+            SRC_PATH.name,
+            c["src_cspace"],
+            c["dst_cspace"],
+            c["range"],
+            c["format"],
+            c["vcodec"],
+            c["ext"],
+        ]
+    )
+
+    src_cspace = c["src_cspace"]
+
+    vf = f"scale=out_range={c['range']}"
+    vf += f",colorspace=all={c['dst_cspace']}"
+    vf += f":iall={src_cspace}"
+    args = [
+        "ffmpeg",
+        "-y",
+        # For input:
+        "-color_primaries",
+        src_cspace,
+        "-color_trc",
+        src_cspace,
+        "-colorspace",
+        src_cspace,
+        "-i",
+        SRC_PATH.as_posix(),
+        # For output:
+        "-bitexact",  # E.g. don't use true random uuids
+        "-vf",
+        vf,
+        "-pix_fmt",
+        c["format"],
+        "-vcodec",
+        c["vcodec"],
+        "-crf",
+        "1",  # Not-quite-lossless
+        (DIR / dst_name).as_posix(),
+    ]
+    if "-v" in ARGS or "-vv" in ARGS:
+        print("$ " + " ".join(args))
+    else:
+        print("  " + args[-1])
+
+    todo.append(args)
+
+# -
+
+with open(DIR / "reftest.list", "r") as f:
+    reftest_list_text = f.read()
+
+for args in todo:
+    vid_name = pathlib.Path(args[-1]).name
+    if vid_name not in reftest_list_text:
+        print(f"WARNING: Not in reftest.list:  {vid_name}")
+
+# -
+
+if "--write" not in ARGS:
+    print("Use --write to write. Exiting...")
+    exit(0)
+
+# -
+
+
+def run_cmd(args):
+    dest = None
+    if "-vv" not in ARGS:
+        dest = subprocess.DEVNULL
+    subprocess.run(args, stderr=dest)
+
+
+with concurrent.futures.ThreadPoolExecutor() as pool:
+    fs = []
+    for cur_args in todo:
+        f = pool.submit(run_cmd, cur_args)
+        fs.append(f)
+
+    done = 0
+    for f in concurrent.futures.as_completed(fs):
+        f.result()  # Raise if it raised
+        done += 1
+        sys.stdout.write(f"\rEncoded {done}/{len(todo)}")

dom/media/test/reftest/image-10bits-rendering-720-90-video.html

@@ -15,5 +15,8 @@
 </head>
 <body>
 <video id="v1" style="position:absolute; left:0; top:0; filter:hue-rotate(90deg);"></video>
+<script>
+//doTest();
+</script>
 </body>
 </html>

dom/media/test/reftest/image-10bits-rendering-90-video.html

@@ -15,5 +15,8 @@
 </head>
 <body>
 <video id="v1" style="position:absolute; left:0; top:0; filter:hue-rotate(90deg);"></video>
+<script>
+//doTest();
+</script>
 </body>
 </html>

dom/media/test/reftest/reftest.list

@@ -1,9 +1,9 @@
-skip-if(Android) fuzzy-if(OSX,0-80,0-76800) fuzzy-if(appleSilicon,92-92,76799-76799) fuzzy-if(winWidget,0-62,0-76799) fuzzy-if(gtkWidget&&layersGPUAccelerated,0-70,0-644) HTTP(..) == short.mp4.firstframe.html short.mp4.firstframe-ref.html
-skip-if(Android) fuzzy-if(OSX,0-87,0-76797) fuzzy-if(appleSilicon,83-83,76797-76797) fuzzy-if(winWidget,0-60,0-76797) fuzzy-if(gtkWidget&&layersGPUAccelerated,0-60,0-1810) HTTP(..) == short.mp4.lastframe.html short.mp4.lastframe-ref.html
-skip-if(Android) skip-if(cocoaWidget) skip-if(winWidget) fuzzy-if(gtkWidget&&layersGPUAccelerated,0-57,0-4281) fuzzy-if(OSX,55-80,4173-4417) HTTP(..) == bipbop_300_215kbps.mp4.lastframe.html bipbop_300_215kbps.mp4.lastframe-ref.html
+skip-if(Android) fuzzy-if(OSX,0-80,0-76800) fuzzy-if(appleSilicon,92-92,76799-76799) fuzzy-if(winWidget,0-62,0-76799) fuzzy-if(gtkWidget&&layersGPUAccelerated,0-70,0-2032) fuzzy-if(swgl,62-69,588-76737) HTTP(..) == short.mp4.firstframe.html short.mp4.firstframe-ref.html
+skip-if(Android) fuzzy-if(OSX,0-87,0-76797) fuzzy-if(appleSilicon,83-83,76797-76797) fuzzy-if(winWidget,0-60,0-76797) fuzzy-if(gtkWidget&&layersGPUAccelerated,0-60,0-6070) fuzzy-if(swgl,52-76,1698-76545) HTTP(..) == short.mp4.lastframe.html short.mp4.lastframe-ref.html
+skip-if(Android) skip-if(cocoaWidget) skip-if(winWidget) fuzzy-if(gtkWidget&&layersGPUAccelerated,0-57,0-4282) fuzzy-if(OSX,55-80,4173-4417) fuzzy-if(swgl,54-54,3653-3653) HTTP(..) == bipbop_300_215kbps.mp4.lastframe.html bipbop_300_215kbps.mp4.lastframe-ref.html
 skip-if(Android) fuzzy-if(OSX,0-25,0-175921) fuzzy-if(appleSilicon,49-49,176063-176063) fuzzy-if(winWidget,0-71,0-179198) fuzzy-if((/^Windows\x20NT\x2010\.0/.test(http.oscpu))&&(/^aarch64-msvc/.test(xulRuntime.XPCOMABI)),0-255,0-179500) HTTP(..) == gizmo.mp4.seek.html gizmo.mp4.55thframe-ref.html
 skip-if(Android) skip-if(MinGW) skip-if((/^Windows\x20NT\x2010\.0/.test(http.oscpu))&&(/^aarch64-msvc/.test(xulRuntime.XPCOMABI))) fuzzy(0-10,0-778236) == image-10bits-rendering-video.html image-10bits-rendering-ref.html
 skip-if(Android) skip-if(MinGW) skip-if((/^Windows\x20NT\x2010\.0/.test(http.oscpu))&&(/^aarch64-msvc/.test(xulRuntime.XPCOMABI))) fuzzy(0-10,0-778536) == image-10bits-rendering-90-video.html image-10bits-rendering-90-ref.html
-skip-if(Android) fuzzy(0-26,0-567562) fuzzy-if(appleSilicon,46-46,575885-575885) == image-10bits-rendering-720-video.html image-10bits-rendering-720-ref.html
+skip-if(Android) fuzzy(0-27,0-573106) fuzzy-if(appleSilicon,46-46,575885-575885) == image-10bits-rendering-720-video.html image-10bits-rendering-720-ref.html
 skip-if(Android) fuzzy(0-31,0-573249) == image-10bits-rendering-720-90-video.html image-10bits-rendering-720-90-ref.html
 skip-if(Android) skip-if(/^Windows\x20NT\x206\.1/.test(http.oscpu)) fuzzy(0-84,0-771156) fails-if(useDrawSnapshot) == uneven_frame_duration_video.html uneven_frame_duration_video-ref.html # Skip on Windows 7 as the resolution of the video is too high for test machines and will fail in the decoder.

dom/media/test/reftest/reftest_img.html

@@ -0,0 +1,20 @@
+<!DOCTYPE HTML>
+<html class="reftest-wait">
+<head>
+<meta charset='utf-8'>
+</head>
+<body>
+<img id="e_img" style="position:absolute; left:0; top:0; max-width:100%">
+<script>
+(async () => {
+  const params = new URLSearchParams(window.location.search);
+  const src = params.get('src');
+  src.defined;
+
+  e_img.src = src;
+  await e_img.decode()
+  document.documentElement.removeAttribute('class');
+})();
+</script>
+</body>
+</html>