Skip to content

Commit

Permalink
[Impeller] Skia gold for flutter_tester dart tests. (flutter#47066)
Browse files Browse the repository at this point in the history
This removes skips for the golden tests in `//testing/dart/canvas_test.dart` and instead passes them up to Skia gold.

Adds a utility class for dealing with Skia gold from these tests, as well as the existing fuzzy identical image comparison for tests that just want to do in memory comparison of images generated from the same test.

Removes the old golden files that were in tree.

Part of flutter/flutter#53784
  • Loading branch information
dnfield authored Nov 2, 2023
1 parent b11e318 commit 9d4c951
Show file tree
Hide file tree
Showing 20 changed files with 244 additions and 85 deletions.
4 changes: 4 additions & 0 deletions .ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,10 @@ targets:
add_recipes_cq: "true"
release_build: "true"
config_name: linux_host_engine
dependencies: >-
[
{"dependency": "goldctl", "version": "git_revision:720a542f6fe4f92922c3b8f0fdcc4d2ac6bb83cd"}
]
drone_dimensions:
- os=Linux

Expand Down
6 changes: 6 additions & 0 deletions ci/builders/linux_host_engine.json
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,12 @@
"device_type=none",
"os=Linux"
],
"dependencies": [
{
"dependency": "goldctl",
"version": "git_revision:720a542f6fe4f92922c3b8f0fdcc4d2ac6bb83cd"
}
],
"gclient_variables": {
"download_android_deps": false
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,16 @@ bool get isLuciEnv => Platform.environment.containsKey(_kLuciEnvName);

/// Fake SkiaGoldClient that is used if the harvester is run outside of Luci.
class FakeSkiaGoldClient implements SkiaGoldClient {
FakeSkiaGoldClient(this._workingDirectory, {this.dimensions});
FakeSkiaGoldClient(this._workingDirectory, {this.dimensions, this.verbose = false});

final Directory _workingDirectory;

@override
final Map<String, String>? dimensions;

@override
final bool verbose;

@override
Future<void> addImg(String testName, File goldenFile,
{double differentPixelsRate = 0.01,
Expand Down
2 changes: 2 additions & 0 deletions impeller/golden_tests_harvester/pubspec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ dependency_overrides:
path: ../../../third_party/dart/third_party/pkg/file/packages/file
meta:
path: ../../../third_party/dart/pkg/meta
engine_repo_tools:
path: ../../tools/pkg/engine_repo_tools
path:
path: ../../../third_party/dart/third_party/pkg/path
platform:
Expand Down
4 changes: 4 additions & 0 deletions lib/web_ui/pubspec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -54,3 +54,7 @@ dev_dependencies:
path: ../../web_sdk/web_engine_tester
skia_gold_client:
path: ../../testing/skia_gold_client

dependency_overrides:
engine_repo_tools:
path: ../../tools/pkg/engine_repo_tools
2 changes: 2 additions & 0 deletions testing/dart/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,11 @@ tests = [
]

foreach(test, tests) {
skia_gold_work_dir = rebase_path("$root_gen_dir/skia_gold_$test")
flutter_frontend_server("compile_$test") {
main_dart = test
kernel_output = "$root_gen_dir/$test.dill"
extra_args = [ "-DkSkiaGoldWorkDirectory=$skia_gold_work_dir" ]
package_config = ".dart_tool/package_config.json"
}
}
Expand Down
90 changes: 14 additions & 76 deletions testing/dart/canvas_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import 'package:litetest/litetest.dart';
import 'package:path/path.dart' as path;
import 'package:vector_math/vector_math_64.dart';

import 'goldens.dart';
import 'impeller_enabled.dart';

typedef CanvasCallback = void Function(Canvas canvas);
Expand Down Expand Up @@ -123,59 +124,9 @@ void testNoCrashes() {
});
}

/// @returns true When the images are reasonably similar.
/// @todo Make the search actually fuzzy to a certain degree.
Future<bool> fuzzyCompareImages(Image golden, Image img) async {
if (golden.width != img.width || golden.height != img.height) {
return false;
}
int getPixel(ByteData data, int x, int y) => data.getUint32((x + y * golden.width) * 4);
final ByteData goldenData = (await golden.toByteData())!;
final ByteData imgData = (await img.toByteData())!;
for (int y = 0; y < golden.height; y++) {
for (int x = 0; x < golden.width; x++) {
if (getPixel(goldenData, x, y) != getPixel(imgData, x, y)) {
return false;
}
}
}
return true;
}

Future<void> saveTestImage(Image image, String filename) async {
final String imagesPath = path.join('flutter', 'testing', 'resources');
final ByteData pngData = (await image.toByteData(format: ImageByteFormat.png))!;
final String outPath = path.join(imagesPath, filename);
File(outPath).writeAsBytesSync(pngData.buffer.asUint8List());
print('wrote: $outPath');
}

/// @returns true When the images are reasonably similar.
Future<bool> fuzzyGoldenImageCompare(
Image image, String goldenImageName) async {
final String imagesPath = path.join('flutter', 'testing', 'resources');
final File file = File(path.join(imagesPath, goldenImageName));

bool areEqual = false;

if (file.existsSync()) {
final Uint8List goldenData = await file.readAsBytes();

final Codec codec = await instantiateImageCodec(goldenData);
final FrameInfo frame = await codec.getNextFrame();
expect(frame.image.height, equals(image.height));
expect(frame.image.width, equals(image.width));

areEqual = await fuzzyCompareImages(frame.image, image);
}
void main() async {
final ImageComparer comparer = await ImageComparer.create();

if (!areEqual) {
saveTestImage(image, 'found_$goldenImageName');
}
return areEqual;
}

void main() {
testNoCrashes();

test('Simple .toImage', () async {
Expand All @@ -190,11 +141,8 @@ void main() {
}, 100, 100);
expect(image.width, equals(100));
expect(image.height, equals(100));

final bool areEqual =
await fuzzyGoldenImageCompare(image, 'canvas_test_toImage.png');
expect(areEqual, true);
}, skip: impellerEnabled);
await comparer.addGoldenImage(image, 'canvas_test_toImage.png');
});

Gradient makeGradient() {
return Gradient.linear(
Expand All @@ -212,10 +160,8 @@ void main() {
expect(image.width, equals(100));
expect(image.height, equals(100));

final bool areEqual =
await fuzzyGoldenImageCompare(image, 'canvas_test_dithered_gradient.png');
expect(areEqual, true);
}, skip: !Platform.isLinux || impellerEnabled); // https://github.com/flutter/flutter/issues/53784
await comparer.addGoldenImage(image, 'canvas_test_dithered_gradient.png');
});

test('Null values allowed for drawAtlas methods', () async {
final Image image = await createImage(100, 100);
Expand Down Expand Up @@ -302,12 +248,8 @@ void main() {
});
}, width, height);

final bool areEqual = await fuzzyCompareImages(incrementalMatrixImage, combinedMatrixImage);
final bool areEqual = await comparer.fuzzyCompareImages(incrementalMatrixImage, combinedMatrixImage);

if (!areEqual) {
saveTestImage(incrementalMatrixImage, 'incremental_3D_transform_test_image.png');
saveTestImage(combinedMatrixImage, 'combined_3D_transform_test_image.png');
}
expect(areEqual, true);
});

Expand Down Expand Up @@ -348,10 +290,8 @@ void main() {
expect(image.width, equals(200));
expect(image.height, equals(250));

final bool areEqual =
await fuzzyGoldenImageCompare(image, 'dotted_path_effect_mixed_with_stroked_geometry.png');
expect(areEqual, true);
}, skip: !Platform.isLinux || impellerEnabled); // https://github.com/flutter/flutter/issues/53784
await comparer.addGoldenImage(image, 'dotted_path_effect_mixed_with_stroked_geometry.png');
});

test('Gradients with matrices in Paragraphs render correctly', () async {
final Image image = await toImage((Canvas canvas) {
Expand Down Expand Up @@ -400,10 +340,8 @@ void main() {
expect(image.width, equals(600));
expect(image.height, equals(400));

final bool areEqual =
await fuzzyGoldenImageCompare(image, 'text_with_gradient_with_matrix.png');
expect(areEqual, true);
}, skip: !Platform.isLinux || impellerEnabled); // https://github.com/flutter/flutter/issues/53784
await comparer.addGoldenImage(image, 'text_with_gradient_with_matrix.png');
});

test('toImageSync - too big', () async {
PictureRecorder recorder = PictureRecorder();
Expand Down Expand Up @@ -602,8 +540,8 @@ void main() {
final Image tofuImage = await drawText('>\b<');

// The tab's image should be identical to the space's image but not the tofu's image.
final bool tabToSpaceComparison = await fuzzyCompareImages(tabImage, spaceImage);
final bool tabToTofuComparison = await fuzzyCompareImages(tabImage, tofuImage);
final bool tabToSpaceComparison = await comparer.fuzzyCompareImages(tabImage, spaceImage);
final bool tabToTofuComparison = await comparer.fuzzyCompareImages(tabImage, tofuImage);

expect(tabToSpaceComparison, isTrue);
expect(tabToTofuComparison, isFalse);
Expand Down
129 changes: 129 additions & 0 deletions testing/dart/goldens.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:io';
import 'dart:typed_data';
import 'dart:ui';

import 'package:path/path.dart' as path;
import 'package:skia_gold_client/skia_gold_client.dart';

import 'impeller_enabled.dart';

const String _kSkiaGoldWorkDirectoryKey = 'kSkiaGoldWorkDirectory';

/// A helper for doing image comparison (golden) tests.
///
/// Contains utilities for comparing two images in memory that are expected to
/// be identical, or for adding images to Skia gold for comparison.
class ImageComparer {
ImageComparer._({
required SkiaGoldClient client,
}) : _client = client;

// Avoid talking to Skia gold for the force-multithreading variants.
static bool get _useSkiaGold =>
!Platform.executableArguments.contains('--force-multithreading');

/// Creates an image comparer and authorizes.
static Future<ImageComparer> create({
bool verbose = false,
}) async {
const String workDirectoryPath =
String.fromEnvironment(_kSkiaGoldWorkDirectoryKey);
if (workDirectoryPath.isEmpty) {
throw UnsupportedError(
'Using ImageComparer requries defining kSkiaGoldWorkDirectoryKey.');
}

final Directory workDirectory = Directory(
impellerEnabled ? '${workDirectoryPath}_iplr' : workDirectoryPath,
)..createSync();
final Map<String, String> dimensions = <String, String>{
'impeller_enabled': impellerEnabled.toString(),
};
final SkiaGoldClient client = isSkiaGoldClientAvailable && _useSkiaGold
? SkiaGoldClient(workDirectory,
dimensions: dimensions, verbose: verbose)
: _FakeSkiaGoldClient(workDirectory, dimensions, verbose: verbose);

await client.auth();
return ImageComparer._(client: client);
}

final SkiaGoldClient _client;

/// Adds an [Image] to Skia Gold for comparison.
///
/// The [fileName] must be unique.
Future<void> addGoldenImage(Image image, String fileName) async {
final ByteData data =
(await image.toByteData(format: ImageByteFormat.png))!;

final File file = File(path.join(_client.workDirectory.path, fileName))
..writeAsBytesSync(data.buffer.asUint8List());
await _client.addImg(
fileName,
file,
screenshotSize: image.width * image.height,
).catchError((dynamic error) {
print('Skia gold comparison failed: $error');
throw Exception('Failed comparison: $fileName');
});
}

Future<bool> fuzzyCompareImages(Image golden, Image testImage) async {
if (golden.width != testImage.width || golden.height != testImage.height) {
return false;
}
int getPixel(ByteData data, int x, int y) =>
data.getUint32((x + y * golden.width) * 4);
final ByteData goldenData = (await golden.toByteData())!;
final ByteData testImageData = (await testImage.toByteData())!;
for (int y = 0; y < golden.height; y++) {
for (int x = 0; x < golden.width; x++) {
if (getPixel(goldenData, x, y) != getPixel(testImageData, x, y)) {
return false;
}
}
}
return true;
}
}

// TODO(dnfield): add local comparison against baseline,
// https://github.com/flutter/flutter/issues/136831
class _FakeSkiaGoldClient implements SkiaGoldClient {
_FakeSkiaGoldClient(
this.workDirectory,
this.dimensions, {
this.verbose = false,
});

@override
final Directory workDirectory;

@override
final Map<String, String> dimensions;

@override
final bool verbose;

@override
Future<void> auth() async {}

@override
Future<void> addImg(
String testName,
File goldenFile, {
double differentPixelsRate = 0.01,
int pixelColorDelta = 0,
required int screenshotSize,
}) async {}

@override
dynamic noSuchMethod(Invocation invocation) {
throw UnimplementedError(invocation.memberName.toString().split('"')[1]);
}
}
15 changes: 15 additions & 0 deletions testing/dart/pubspec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ environment:
dependencies:
litetest: any
path: any
skia_gold_client: any
sky_engine: any
vector_math: any
vm_service: any
Expand All @@ -29,8 +30,14 @@ dependency_overrides:
path: ../../../third_party/dart/pkg/async_helper
collection:
path: ../../../third_party/dart/third_party/pkg/collection
crypto:
path: ../../../third_party/dart/third_party/pkg/crypto
engine_repo_tools:
path: ../../tools/pkg/engine_repo_tools
expect:
path: ../../../third_party/dart/pkg/expect
file:
path: ../../../third_party/dart/third_party/pkg/file/packages/file
fixnum:
path: ../../../third_party/dart/third_party/pkg/fixnum
litetest:
Expand All @@ -39,12 +46,20 @@ dependency_overrides:
path: ../../../third_party/dart/pkg/meta
path:
path: ../../../third_party/dart/third_party/pkg/path
platform:
path: ../../../third_party/pkg/platform
process:
path: ../../../third_party/pkg/process
protobuf:
path: ../../../third_party/dart/third_party/pkg/protobuf/protobuf
smith:
path: ../../../third_party/dart/pkg/smith
skia_gold_client:
path: ../skia_gold_client
sky_engine:
path: ../../sky/packages/sky_engine
typed_data:
path: ../../../third_party/dart/third_party/pkg/typed_data
vector_math:
path: ../../../third_party/pkg/vector_math
vm_service:
Expand Down
Binary file removed testing/resources/canvas_test_dithered_gradient.png
Binary file not shown.
Binary file removed testing/resources/canvas_test_gradient.png
Binary file not shown.
Binary file removed testing/resources/canvas_test_toImage.png
Binary file not shown.
Binary file not shown.
Binary file not shown.
Loading

0 comments on commit 9d4c951

Please sign in to comment.