[firebase_ml_vision] adds facial contours (firebase#1599)

.gitignore

@@ -35,3 +35,7 @@ GeneratedPluginRegistrant.m
 GeneratedPluginRegistrant.java
 build/
 .flutter-plugins
+
+.project
+.classpath
+.settings

packages/firebase_ml_vision/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.9.2
+
+* Add detection of `FaceContour`s when using the `FaceDetector`. See `README.md` for more information.
+
 ## 0.9.1+1
 
 * Update google-services Android gradle plugin to 4.3.0 in documentation and examples.

packages/firebase_ml_vision/README.md

@@ -25,6 +25,18 @@ android {
 }
 ```
 
+If you're using the on-device `Face Contour Detection`, include the latest matching [ML Kit: Face Detection Model](https://firebase.google.com/support/release-notes/android) dependency in your app-level build.gradle file.
+
+```
+android {
+    dependencies {
+        // ...
+
+        api 'com.google.firebase:firebase-ml-vision-face-model:17.0.2'
+    }
+}
+```
+
 If you receive compilation errors, try an earlier version of [ML Kit: Image Labeling](https://firebase.google.com/support/release-notes/android).
 
 Optional but recommended: If you use the on-device API, configure your app to automatically download the ML model to the device after your app is installed from the Play Store. To do so, add the following declaration to your app's AndroidManifest.xml file:

packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java

@@ -5,7 +5,9 @@
 import com.google.android.gms.tasks.OnSuccessListener;
 import com.google.firebase.ml.vision.FirebaseVision;
 import com.google.firebase.ml.vision.common.FirebaseVisionImage;
+import com.google.firebase.ml.vision.common.FirebaseVisionPoint;
 import com.google.firebase.ml.vision.face.FirebaseVisionFace;
+import com.google.firebase.ml.vision.face.FirebaseVisionFaceContour;
 import com.google.firebase.ml.vision.face.FirebaseVisionFaceDetector;
 import com.google.firebase.ml.vision.face.FirebaseVisionFaceDetectorOptions;
 import com.google.firebase.ml.vision.face.FirebaseVisionFaceLandmark;
@@ -63,6 +65,8 @@ public void onSuccess(List<FirebaseVisionFace> firebaseVisionFaces) {
 
                   faceData.put("landmarks", getLandmarkData(face));
 
+                  faceData.put("contours", getContourData(face));
+
                   faces.add(faceData);
                 }
 
@@ -95,6 +99,34 @@ private Map<String, double[]> getLandmarkData(FirebaseVisionFace face) {
     return landmarks;
   }
 
+  private Map<String, List<double[]>> getContourData(FirebaseVisionFace face) {
+    Map<String, List<double[]>> contours = new HashMap<>();
+
+    contours.put("allPoints", contourPosition(face, FirebaseVisionFaceContour.ALL_POINTS));
+    contours.put("face", contourPosition(face, FirebaseVisionFaceContour.FACE));
+    contours.put("leftEye", contourPosition(face, FirebaseVisionFaceContour.LEFT_EYE));
+    contours.put(
+        "leftEyebrowBottom", contourPosition(face, FirebaseVisionFaceContour.LEFT_EYEBROW_BOTTOM));
+    contours.put(
+        "leftEyebrowTop", contourPosition(face, FirebaseVisionFaceContour.LEFT_EYEBROW_TOP));
+    contours.put(
+        "lowerLipBottom", contourPosition(face, FirebaseVisionFaceContour.LOWER_LIP_BOTTOM));
+    contours.put("lowerLipTop", contourPosition(face, FirebaseVisionFaceContour.LOWER_LIP_TOP));
+    contours.put("noseBottom", contourPosition(face, FirebaseVisionFaceContour.NOSE_BOTTOM));
+    contours.put("noseBridge", contourPosition(face, FirebaseVisionFaceContour.NOSE_BRIDGE));
+    contours.put("rightEye", contourPosition(face, FirebaseVisionFaceContour.RIGHT_EYE));
+    contours.put(
+        "rightEyebrowBottom",
+        contourPosition(face, FirebaseVisionFaceContour.RIGHT_EYEBROW_BOTTOM));
+    contours.put(
+        "rightEyebrowTop", contourPosition(face, FirebaseVisionFaceContour.RIGHT_EYEBROW_TOP));
+    contours.put(
+        "upperLipBottom", contourPosition(face, FirebaseVisionFaceContour.UPPER_LIP_BOTTOM));
+    contours.put("upperLipTop", contourPosition(face, FirebaseVisionFaceContour.UPPER_LIP_TOP));
+
+    return contours;
+  }
+
   private double[] landmarkPosition(FirebaseVisionFace face, int landmarkInt) {
     FirebaseVisionFaceLandmark landmark = face.getLandmark(landmarkInt);
     if (landmark != null) {
@@ -104,6 +136,22 @@ private double[] landmarkPosition(FirebaseVisionFace face, int landmarkInt) {
     return null;
   }
 
+  private List<double[]> contourPosition(FirebaseVisionFace face, int contourInt) {
+    FirebaseVisionFaceContour contour = face.getContour(contourInt);
+    if (contour != null) {
+      List<FirebaseVisionPoint> contourPoints = contour.getPoints();
+      List<double[]> result = new ArrayList<double[]>();
+
+      for (int i = 0; i < contourPoints.size(); i++) {
+        result.add(new double[] {contourPoints.get(i).getX(), contourPoints.get(i).getY()});
+      }
+
+      return result;
+    }
+
+    return null;
+  }
+
   private FirebaseVisionFaceDetectorOptions parseOptions(Map<String, Object> options) {
     int classification =
         (boolean) options.get("enableClassification")
@@ -115,6 +163,11 @@ private FirebaseVisionFaceDetectorOptions parseOptions(Map<String, Object> optio
             ? FirebaseVisionFaceDetectorOptions.ALL_LANDMARKS
             : FirebaseVisionFaceDetectorOptions.NO_LANDMARKS;
 
+    int contours =
+        (boolean) options.get("enableContours")
+            ? FirebaseVisionFaceDetectorOptions.ALL_CONTOURS
+            : FirebaseVisionFaceDetectorOptions.NO_CONTOURS;
+
     int mode;
     switch ((String) options.get("mode")) {
       case "accurate":
@@ -131,6 +184,7 @@ private FirebaseVisionFaceDetectorOptions parseOptions(Map<String, Object> optio
         new FirebaseVisionFaceDetectorOptions.Builder()
             .setClassificationMode(classification)
             .setLandmarkMode(landmark)
+            .setContourMode(contours)
             .setMinFaceSize((float) ((double) options.get("minFaceSize")))
             .setPerformanceMode(mode);
 

packages/firebase_ml_vision/example/android/app/build.gradle

@@ -48,6 +48,7 @@ android {
 
     dependencies {
         api 'com.google.firebase:firebase-ml-vision-image-label-model:17.0.2'
+        api 'com.google.firebase:firebase-ml-vision-face-model:17.0.2'
     }
 }
 

packages/firebase_ml_vision/example/test_driver/face_detector.dart

@@ -6,7 +6,10 @@ part of 'firebase_ml_vision.dart';
 
 void faceDetectorTests() {
   group('$FaceDetector', () {
-    final FaceDetector detector = FirebaseVision.instance.faceDetector();
+    final FaceDetector detector = FirebaseVision.instance.faceDetector(
+      FaceDetectorOptions(
+          enableContours: true, mode: FaceDetectorMode.accurate),
+    );
 
     test('processImage', () async {
       final String tmpFilename = await _loadImage('assets/test_face.jpg');
@@ -16,6 +19,10 @@ void faceDetectorTests() {
       final List<Face> faces = await detector.processImage(visionImage);
 
       expect(faces.length, 1);
+      expect(
+        faces[0].getContour(FaceContourType.allPoints).positionsList,
+        isNotEmpty,
+      );
     });
 
     test('close', () {

packages/firebase_ml_vision/ios/Classes/FaceDetector.m

@@ -68,6 +68,35 @@ - (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result {
                 @"rightMouth" : [FaceDetector getLandmarkPosition:face
                                                          landmark:FIRFaceLandmarkTypeMouthRight],
               },
+              @"contours" : @{
+                @"allPoints" : [FaceDetector getContourPoints:face contour:FIRFaceContourTypeAll],
+                @"face" : [FaceDetector getContourPoints:face contour:FIRFaceContourTypeFace],
+                @"leftEye" : [FaceDetector getContourPoints:face contour:FIRFaceContourTypeLeftEye],
+                @"leftEyebrowBottom" :
+                    [FaceDetector getContourPoints:face
+                                           contour:FIRFaceContourTypeLeftEyebrowBottom],
+                @"leftEyebrowTop" :
+                    [FaceDetector getContourPoints:face contour:FIRFaceContourTypeLeftEyebrowTop],
+                @"lowerLipBottom" :
+                    [FaceDetector getContourPoints:face contour:FIRFaceContourTypeLowerLipBottom],
+                @"lowerLipTop" : [FaceDetector getContourPoints:face
+                                                        contour:FIRFaceContourTypeLowerLipTop],
+                @"noseBottom" : [FaceDetector getContourPoints:face
+                                                       contour:FIRFaceContourTypeNoseBottom],
+                @"noseBridge" : [FaceDetector getContourPoints:face
+                                                       contour:FIRFaceContourTypeNoseBridge],
+                @"rightEye" : [FaceDetector getContourPoints:face
+                                                     contour:FIRFaceContourTypeRightEye],
+                @"rightEyebrowBottom" :
+                    [FaceDetector getContourPoints:face
+                                           contour:FIRFaceContourTypeRightEyebrowBottom],
+                @"rightEyebrowTop" :
+                    [FaceDetector getContourPoints:face contour:FIRFaceContourTypeRightEyebrowTop],
+                @"upperLipBottom" :
+                    [FaceDetector getContourPoints:face contour:FIRFaceContourTypeUpperLipBottom],
+                @"upperLipTop" : [FaceDetector getContourPoints:face
+                                                        contour:FIRFaceContourTypeUpperLipTop],
+              }
             };
 
             [faceData addObject:data];
@@ -86,6 +115,21 @@ + (id)getLandmarkPosition:(FIRVisionFace *)face landmark:(FIRFaceLandmarkType)la
   return [NSNull null];
 }
 
++ (id)getContourPoints:(FIRVisionFace *)face contour:(FIRFaceContourType)contourType {
+  FIRVisionFaceContour *contour = [face contourOfType:contourType];
+  if (contour) {
+    NSArray<FIRVisionPoint *> *contourPoints = contour.points;
+    NSMutableArray *result = [[NSMutableArray alloc] initWithCapacity:[contourPoints count]];
+    for (int i = 0; i < [contourPoints count]; i++) {
+      FIRVisionPoint *point = [contourPoints objectAtIndex:i];
+      [result insertObject:@[ point.x, point.y ] atIndex:i];
+    }
+    return [result copy];
+  }
+
+  return [NSNull null];
+}
+
 + (FIRVisionFaceDetectorOptions *)parseOptions:(NSDictionary *)optionsData {
   FIRVisionFaceDetectorOptions *options = [[FIRVisionFaceDetectorOptions alloc] init];
 
@@ -103,6 +147,13 @@ + (FIRVisionFaceDetectorOptions *)parseOptions:(NSDictionary *)optionsData {
     options.landmarkMode = FIRVisionFaceDetectorLandmarkModeNone;
   }
 
+  NSNumber *enableContours = optionsData[@"enableContours"];
+  if (enableContours.boolValue) {
+    options.contourMode = FIRVisionFaceDetectorContourModeAll;
+  } else {
+    options.contourMode = FIRVisionFaceDetectorContourModeNone;
+  }
+
   NSNumber *enableTracking = optionsData[@"enableTracking"];
   options.trackingEnabled = enableTracking.boolValue;
 

packages/firebase_ml_vision/lib/src/face_detector.dart

@@ -24,6 +24,24 @@ enum FaceLandmarkType {
   rightMouth,
 }
 
+/// Available face contour types detected by [FaceDetector].
+enum FaceContourType {
+  allPoints,
+  face,
+  leftEye,
+  leftEyebrowBottom,
+  leftEyebrowTop,
+  lowerLipBottom,
+  lowerLipTop,
+  noseBottom,
+  noseBridge,
+  rightEye,
+  rightEyebrowBottom,
+  rightEyebrowTop,
+  upperLipBottom,
+  upperLipTop
+}
+
 /// Detector for detecting faces in an input image.
 ///
 /// A face detector is created via
@@ -59,6 +77,7 @@ class FaceDetector {
         'options': <String, dynamic>{
           'enableClassification': options.enableClassification,
           'enableLandmarks': options.enableLandmarks,
+          'enableContours': options.enableContours,
           'enableTracking': options.enableTracking,
           'minFaceSize': options.minFaceSize,
           'mode': _enumToString(options.mode),
@@ -98,6 +117,7 @@ class FaceDetectorOptions {
   const FaceDetectorOptions({
     this.enableClassification = false,
     this.enableLandmarks = false,
+    this.enableContours = false,
     this.enableTracking = false,
     this.minFaceSize = 0.1,
     this.mode = FaceDetectorMode.fast,
@@ -112,6 +132,9 @@ class FaceDetectorOptions {
   /// Whether to detect [FaceLandmark]s.
   final bool enableLandmarks;
 
+  /// Whether to detect [FaceContour]s.
+  final bool enableContours;
+
   /// Whether to enable face tracking.
   ///
   /// If enabled, the detector will maintain a consistent ID for each face when
@@ -154,9 +177,25 @@ class Face {
                   type,
                   Offset(pos[0], pos[1]),
                 );
+        })),
+        _contours = Map<FaceContourType, FaceContour>.fromIterables(
+            FaceContourType.values,
+            FaceContourType.values.map((FaceContourType type) {
+          /// added empty map to pass the tests
+          final List<dynamic> arr =
+              (data['contours'] ?? <String, dynamic>{})[_enumToString(type)];
+          return (arr == null)
+              ? null
+              : FaceContour._(
+                  type,
+                  arr
+                      .map<Offset>((dynamic pos) => Offset(pos[0], pos[1]))
+                      .toList(),
+                );
         }));
 
   final Map<FaceLandmarkType, FaceLandmark> _landmarks;
+  final Map<FaceContourType, FaceContour> _contours;
 
   /// The axis-aligned bounding rectangle of the detected face.
   ///
@@ -212,6 +251,11 @@ class Face {
   ///
   /// Null if landmark was not detected.
   FaceLandmark getLandmark(FaceLandmarkType landmark) => _landmarks[landmark];
+
+  /// Gets the contour based on the provided [FaceContourType].
+  ///
+  /// Null if contour was not detected.
+  FaceContour getContour(FaceContourType contour) => _contours[contour];
 }
 
 /// Represent a face landmark.
@@ -228,3 +272,18 @@ class FaceLandmark {
   /// The point (0, 0) is defined as the upper-left corner of the image.
   final Offset position;
 }
+
+/// Represent a face contour.
+///
+/// Contours of facial features.
+class FaceContour {
+  FaceContour._(this.type, this.positionsList);
+
+  /// The [FaceContourType] of this contour.
+  final FaceContourType type;
+
+  /// Gets a 2D point [List] for contour positions.
+  ///
+  /// The point (0, 0) is defined as the upper-left corner of the image.
+  final List<Offset> positionsList;
+}

packages/firebase_ml_vision/pubspec.yaml

@@ -2,7 +2,7 @@ name: firebase_ml_vision
 description: Flutter plugin for Firebase machine learning vision services.
 author: Flutter Team <[email protected]>
 homepage: https://github.com/flutter/plugins/tree/master/packages/firebase_ml_vision
-version: 0.9.1+1
+version: 0.9.2
 
 dependencies:
   flutter: