diff --git a/lib/web_ui/dev/test_platform.dart b/lib/web_ui/dev/test_platform.dart index d9fd05ccd6efd..cc2dd887e1af3 100644 --- a/lib/web_ui/dev/test_platform.dart +++ b/lib/web_ui/dev/test_platform.dart @@ -339,17 +339,20 @@ class BrowserPlatform extends PlatformPlugin { requestData['region'] as Map; final PixelComparison pixelComparison = PixelComparison.values.firstWhere( (PixelComparison value) => value.toString() == requestData['pixelComparison']); + final bool isCanvaskitTest = requestData['isCanvaskitTest'] as bool; final String result = await _diffScreenshot( - filename, write, maxDiffRate, region, pixelComparison); + filename, write, maxDiffRate, region, pixelComparison, isCanvaskitTest); return shelf.Response.ok(json.encode(result)); } Future _diffScreenshot( - String filename, - bool write, - double maxDiffRateFailure, - Map region, - PixelComparison pixelComparison) async { + String filename, + bool write, + double maxDiffRateFailure, + Map region, + PixelComparison pixelComparison, + bool isCanvaskitTest, + ) async { if (doUpdateScreenshotGoldens) { write = true; } @@ -387,6 +390,7 @@ class BrowserPlatform extends PlatformPlugin { pixelComparison, maxDiffRateFailure, skiaClient, + isCanvaskitTest: isCanvaskitTest, goldensDirectory: goldensDirectory, filenameSuffix: _screenshotManager!.filenameSuffix, write: write, diff --git a/web_sdk/web_engine_tester/lib/golden_tester.dart b/web_sdk/web_engine_tester/lib/golden_tester.dart index fcd22fedf8136..db7b0dc3188a2 100644 --- a/web_sdk/web_engine_tester/lib/golden_tester.dart +++ b/web_sdk/web_engine_tester/lib/golden_tester.dart @@ -6,10 +6,9 @@ import 'dart:async'; import 'dart:convert'; import 'dart:html' as html; -import 'package:ui/src/engine.dart'; -import 'package:ui/ui.dart'; - import 'package:test/test.dart'; +import 'package:ui/src/engine.dart' show operatingSystem, OperatingSystem, useCanvasKit; +import 'package:ui/ui.dart'; Future _callScreenshotServer(dynamic requestData) async { final html.HttpRequest request = await html.HttpRequest.request( @@ -63,6 +62,7 @@ Future matchGoldenFile(String filename, 'height': region.height }, 'pixelComparison': pixelComparison.toString(), + 'isCanvaskitTest': useCanvasKit, }; // Chrome on macOS renders slightly differently from Linux, so allow it an diff --git a/web_sdk/web_test_utils/lib/image_compare.dart b/web_sdk/web_test_utils/lib/image_compare.dart index 7952930c95658..4a3bef917f034 100644 --- a/web_sdk/web_test_utils/lib/image_compare.dart +++ b/web_sdk/web_test_utils/lib/image_compare.dart @@ -32,6 +32,7 @@ Future compareImage( PixelComparison pixelComparison, double maxDiffRateFailure, SkiaGoldClient? skiaClient, { + required bool isCanvaskitTest, // TODO(mdebbar): Remove these args with goldens repo. String goldensDirectory = '', String filenameSuffix = '', @@ -43,7 +44,7 @@ Future compareImage( // comparison. // TODO(mdebbar): Use Skia Gold for comparison, not only for uploading. - await _uploadToSkiaGold(skiaClient, screenshot, filename); + await _uploadToSkiaGold(skiaClient, screenshot, filename, isCanvaskitTest); } filename = filename.replaceAll('.png', '$filenameSuffix.png'); @@ -163,6 +164,7 @@ Future _uploadToSkiaGold( SkiaGoldClient skiaClient, Image screenshot, String filename, + bool isCanvaskitTest, ) async { // Can't upload to Gold Skia unless running in LUCI. assert(_isLuci); @@ -172,11 +174,13 @@ Future _uploadToSkiaGold( final File goldenFile = File(p.join(environment.webUiSkiaGoldDirectory.path, filename)); await goldenFile.writeAsBytes(encodePng(screenshot), flush: true); + final int screenshotSize = screenshot.width * screenshot.height; + if (_isPreSubmit) { - return _uploadInPreSubmit(skiaClient, filename, goldenFile); + return _uploadInPreSubmit(skiaClient, filename, goldenFile, screenshotSize, isCanvaskitTest); } if (_isPostSubmit) { - return _uploadInPostSubmit(skiaClient, filename, goldenFile); + return _uploadInPostSubmit(skiaClient, filename, goldenFile, screenshotSize, isCanvaskitTest); } } @@ -184,16 +188,20 @@ Future _uploadInPreSubmit( SkiaGoldClient skiaClient, String filename, File goldenFile, + int screenshotSize, + bool isCanvaskitTest, ) { assert(_isPreSubmit); - return skiaClient.tryjobAdd(filename, goldenFile); + return skiaClient.tryjobAdd(filename, goldenFile, screenshotSize, isCanvaskitTest); } Future _uploadInPostSubmit( SkiaGoldClient skiaClient, String filename, File goldenFile, + int screenshotSize, + bool isCanvaskitTest, ) { assert(_isPostSubmit); - return skiaClient.imgtestAdd(filename, goldenFile); + return skiaClient.imgtestAdd(filename, goldenFile, screenshotSize, isCanvaskitTest); } diff --git a/web_sdk/web_test_utils/lib/skia_client.dart b/web_sdk/web_test_utils/lib/skia_client.dart index fae028364bcd9..13aeb7c16acda 100644 --- a/web_sdk/web_test_utils/lib/skia_client.dart +++ b/web_sdk/web_test_utils/lib/skia_client.dart @@ -16,6 +16,13 @@ const String _kGoldctlKey = 'GOLDCTL'; const String _skiaGoldHost = 'https://flutter-engine-gold.skia.org'; const String _instance = 'flutter-engine'; +/// The percentage of accepted pixels to be wrong. +/// +/// This should be a double between 0.0 and 1.0. A value of 0.0 means we don't +/// accept any pixel to be different. A value of 1.0 means we accept 100% of +/// pixels to be different. +const double kMaxDifferentPixelsRate = 0.1; + /// A client for uploading image tests and making baseline requests to the /// Flutter Gold Dashboard. class SkiaGoldClient { @@ -162,7 +169,12 @@ class SkiaGoldClient { /// /// The [testName] and [goldenFile] parameters reference the current /// comparison being evaluated. - Future imgtestAdd(String testName, File goldenFile) async { + Future imgtestAdd( + String testName, + File goldenFile, + int screenshotSize, + bool isCanvaskitTest, + ) async { await _imgtestInit(); final List imgtestCommand = [ @@ -171,6 +183,7 @@ class SkiaGoldClient { '--work-dir', _tempPath, '--test-name', cleanTestName(testName), '--png-file', goldenFile.path, + ..._getMatchingArguments(testName, screenshotSize, isCanvaskitTest), ]; final ProcessResult result = await process.run(imgtestCommand); @@ -250,7 +263,12 @@ class SkiaGoldClient { /// /// The [testName] and [goldenFile] parameters reference the current /// comparison being evaluated. - Future tryjobAdd(String testName, File goldenFile) async { + Future tryjobAdd( + String testName, + File goldenFile, + int screenshotSize, + bool isCanvaskitTest, + ) async { await _tryjobInit(); final List imgtestCommand = [ @@ -259,6 +277,7 @@ class SkiaGoldClient { '--work-dir', _tempPath, '--test-name', cleanTestName(testName), '--png-file', goldenFile.path, + ..._getMatchingArguments(testName, screenshotSize, isCanvaskitTest), ]; final ProcessResult result = await process.run(imgtestCommand); @@ -279,6 +298,44 @@ class SkiaGoldClient { } } + List _getMatchingArguments( + String testName, + int screenshotSize, + bool isCanvaskitTest, + ) { + // The algorithm to be used when matching images. The available options are: + // - "fuzzy": Allows for customizing the thresholds of pixel differences. + // - "sobel": Same as "fuzzy" but performs edge detection before performing + // a fuzzy match. + const String algorithm = 'fuzzy'; + + // The number of pixels in this image that are allowed to differ from the + // baseline. It's okay for this to be a slightly high number like 10% of the + // image size because those wrong pixels are constrained by + // `pixelDeltaThreshold` below. + final int maxDifferentPixels = (screenshotSize * kMaxDifferentPixelsRate).toInt(); + + // The maximum acceptable difference in RGB channels of each pixel. + // + // ``` + // abs(r(image) - r(golden)) + abs(g(image) - g(golden)) + abs(b(image) - b(golden)) <= pixelDeltaThreshold + // ``` + final String pixelDeltaThreshold; + if (isCanvaskitTest) { + pixelDeltaThreshold = '21'; + } else if (browserName == 'ios-safari') { + pixelDeltaThreshold = '15'; + } else { + pixelDeltaThreshold = '3'; + } + + return [ + '--add-test-optional-key', 'image_matching_algorithm:$algorithm', + '--add-test-optional-key', 'fuzzy_max_different_pixels:$maxDifferentPixels', + '--add-test-optional-key', 'fuzzy_pixel_delta_threshold:$pixelDeltaThreshold', + ]; + } + /// Returns the latest positive digest for the given test known to Skia Gold /// at head. Future getExpectationForTest(String testName) async { @@ -356,9 +413,9 @@ class SkiaGoldClient { /// browser the image was rendered on. Map _getKeys() { return { + 'Browser': browserName, 'CI': 'luci', 'Platform': Platform.operatingSystem, - 'Browser': browserName, }; }