feat: implement DetectionResultAnnotationController (homuler#983)

* fix: hide Label when the text is empty

* feat: implement DetectionResultAnnotationController

Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/AnnotationController.cs

@@ -50,6 +50,8 @@ public RotationAngle rotationAngle
       }
     }
 
+    public Vector2Int imageSize { get; set; }
+
     protected virtual void Start()
     {
       if (!TryGetComponent<RectTransform>(out var _))

Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/DetectionAnnotation.cs

@@ -6,6 +6,9 @@
 
 using Mediapipe.Unity.CoordinateSystem;
 using UnityEngine;
+using UnityEngine.Serialization;
+
+using mptcc = Mediapipe.Tasks.Components.Containers;
 
 namespace Mediapipe.Unity
 {
@@ -15,16 +18,18 @@ namespace Mediapipe.Unity
 
   public sealed class DetectionAnnotation : HierarchicalAnnotation
   {
-    [SerializeField] private RectangleAnnotation _locationDataAnnotation;
-    [SerializeField] private PointListAnnotation _relativeKeypointsAnnotation;
+    [FormerlySerializedAs("_locationDataAnnotation")]
+    [SerializeField] private RectangleAnnotation _boundingBoxAnnotation;
+    [FormerlySerializedAs("_relativeKeypointsAnnotation")]
+    [SerializeField] private PointListAnnotation _keypointsAnnotation;
     [SerializeField] private LabelAnnotation _labelAnnotation;
 
     public override bool isMirrored
     {
       set
       {
-        _locationDataAnnotation.isMirrored = value;
-        _relativeKeypointsAnnotation.isMirrored = value;
+        _boundingBoxAnnotation.isMirrored = value;
+        _keypointsAnnotation.isMirrored = value;
         _labelAnnotation.isMirrored = value;
         base.isMirrored = value;
       }
@@ -34,21 +39,21 @@ public override RotationAngle rotationAngle
     {
       set
       {
-        _locationDataAnnotation.rotationAngle = value;
-        _relativeKeypointsAnnotation.rotationAngle = value;
+        _boundingBoxAnnotation.rotationAngle = value;
+        _keypointsAnnotation.rotationAngle = value;
         _labelAnnotation.rotationAngle = value;
         base.rotationAngle = value;
       }
     }
 
     public void SetLineWidth(float lineWidth)
     {
-      _locationDataAnnotation.SetLineWidth(lineWidth);
+      _boundingBoxAnnotation.SetLineWidth(lineWidth);
     }
 
     public void SetKeypointRadius(float radius)
     {
-      _relativeKeypointsAnnotation.SetRadius(radius);
+      _keypointsAnnotation.SetRadius(radius);
     }
 
     /// <param name="threshold">
@@ -66,8 +71,8 @@ public void Draw(Detection target, float threshold = 0.0f)
         // Assume that location data's format is always RelativeBoundingBox
         // TODO: fix if there are cases where this assumption is not correct.
         var rectVertices = GetScreenRect().GetRectVertices(target.LocationData.RelativeBoundingBox, rotationAngle, isMirrored);
-        _locationDataAnnotation.SetColor(GetColor(score, Mathf.Clamp(threshold, 0.0f, 1.0f)));
-        _locationDataAnnotation.Draw(rectVertices);
+        _boundingBoxAnnotation.SetColor(GetColor(score, Mathf.Clamp(threshold, 0.0f, 1.0f)));
+        _boundingBoxAnnotation.Draw(rectVertices);
 
         var width = rectVertices[2].x - rectVertices[0].x;
         var height = rectVertices[2].y - rectVertices[0].y;
@@ -77,7 +82,36 @@ public void Draw(Detection target, float threshold = 0.0f)
         var (maxWidth, maxHeight) = isInverted ? (height, width) : (width, height);
         _labelAnnotation.Draw(labelText, rectVertices[vertexId], color, maxWidth, maxHeight);
 
-        _relativeKeypointsAnnotation.Draw(target.LocationData.RelativeKeypoints);
+        _keypointsAnnotation.Draw(target.LocationData.RelativeKeypoints);
+      }
+    }
+
+    /// <param name="threshold">
+    ///   Score threshold. This value must be between 0 and 1.
+    ///   This will affect the rectangle's color. For example, if the score is below the threshold, the rectangle will be transparent.
+    ///   The default value is 0.
+    /// </param>
+    public void Draw(mptcc.Detection target, Vector2Int imageSize, float threshold = 0.0f)
+    {
+      if (ActivateFor(target))
+      {
+        var category = target.categories?.Count > 0 ? (mptcc.Category?)target.categories[0] : null;
+        var score = category != null ? category.Value.score : 1.0f;
+        var color = GetColor(score, Mathf.Clamp(threshold, 0.0f, 1.0f));
+
+        var rectVertices = GetScreenRect().GetRectVertices(target.boundingBox, imageSize, rotationAngle, isMirrored);
+        _boundingBoxAnnotation.SetColor(GetColor(score, Mathf.Clamp(threshold, 0.0f, 1.0f)));
+        _boundingBoxAnnotation.Draw(rectVertices);
+
+        var width = rectVertices[2].x - rectVertices[0].x;
+        var height = rectVertices[2].y - rectVertices[0].y;
+        var labelText = category?.categoryName;
+        var vertexId = (((int)rotationAngle / 90) + 1) % 4;
+        var isInverted = ImageCoordinate.IsInverted(rotationAngle);
+        var (maxWidth, maxHeight) = isInverted ? (height, width) : (width, height);
+        _labelAnnotation.Draw(labelText, rectVertices[vertexId], color, maxWidth, maxHeight);
+
+        _keypointsAnnotation.Draw(target.keypoints);
       }
     }
 

Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/DetectionListAnnotation.cs

@@ -7,6 +7,8 @@
 using System.Collections.Generic;
 using UnityEngine;
 
+using mptcc = Mediapipe.Tasks.Components.Containers;
+
 namespace Mediapipe.Unity
 {
   public sealed class DetectionListAnnotation : ListAnnotation<DetectionAnnotation>
@@ -37,6 +39,32 @@ public void SetKeypointRadius(float keypointRadius)
       ApplyKeypointRadius(keypointRadius);
     }
 
+    /// <param name="threshold">
+    ///   Score threshold. This value must be between 0 and 1.
+    ///   This will affect the rectangle's color. For example, if the score is below the threshold, the rectangle will be transparent.
+    ///   The default value is 0.
+    /// </param>
+    public void Draw(IReadOnlyList<mptcc.Detection> targets, Vector2Int imageSize, float threshold = 0.0f)
+    {
+      if (ActivateFor(targets))
+      {
+        CallActionForAll(targets, (annotation, target) =>
+        {
+          if (annotation != null) { annotation.Draw(target, imageSize, threshold); }
+        });
+      }
+    }
+
+    /// <param name="threshold">
+    ///   Score threshold. This value must be between 0 and 1.
+    ///   This will affect the rectangle's color. For example, if the score is below the threshold, the rectangle will be transparent.
+    ///   The default value is 0.
+    /// </param>
+    public void Draw(mptcc.DetectionResult target, Vector2Int imageSize, float threshold = 0.0f)
+    {
+      Draw(target.detections, imageSize, threshold);
+    }
+
     /// <param name="threshold">
     ///   Score threshold. This value must be between 0 and 1.
     ///   This will affect the rectangle's color. For example, if the score is below the threshold, the rectangle will be transparent.

Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/DetectionResultAnnotationController.cs

@@ -0,0 +1,33 @@
+// Copyright (c) 2023 homuler
+//
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT.
+
+using UnityEngine;
+
+using mptcc = Mediapipe.Tasks.Components.Containers;
+
+namespace Mediapipe.Unity
+{
+  public class DetectionResultAnnotationController : AnnotationController<DetectionListAnnotation>
+  {
+    [SerializeField, Range(0, 1)] private float _threshold = 0.0f;
+
+    private mptcc.DetectionResult _currentTarget;
+
+    public void DrawNow(mptcc.DetectionResult target)
+    {
+      _currentTarget = target;
+      SyncNow();
+    }
+
+    public void DrawLater(mptcc.DetectionResult target) => UpdateCurrentTarget(target, ref _currentTarget);
+
+    protected override void SyncNow()
+    {
+      isStale = false;
+      annotation.Draw(_currentTarget, imageSize, _threshold);
+    }
+  }
+}

Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/LabelAnnotation.cs

@@ -66,5 +66,7 @@ private float CalcContrastRatio(Color lighter, Color darker)
       var l2 = CalcRelativeLuminance(darker);
       return (l1 + 0.05f) / (l2 + 0.05f);
     }
+
+    private bool ActivateFor(string text) => base.ActivateFor(string.IsNullOrEmpty(text) ? null : text);
   }
 }

Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/PointAnnotation.cs

@@ -8,6 +8,7 @@
 using UnityEngine;
 
 using mplt = Mediapipe.LocationData.Types;
+using mptcc = Mediapipe.Tasks.Components.Containers;
 
 namespace Mediapipe.Unity
 {
@@ -84,6 +85,15 @@ public void Draw(mplt.RelativeKeypoint target, float threshold = 0.0f)
       }
     }
 
+    public void Draw(mptcc.NormalizedKeypoint target, float threshold = 0.0f)
+    {
+      if (ActivateFor(target))
+      {
+        Draw(GetScreenRect().GetPoint(target, rotationAngle, isMirrored));
+        SetColor(GetColor(target.score ?? 1.0f, threshold));
+      }
+    }
+
     private void ApplyColor(Color color)
     {
       GetComponent<Renderer>().material.color = color;

Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/PointListAnnotation.cs

@@ -8,6 +8,7 @@
 using UnityEngine;
 
 using mplt = Mediapipe.LocationData.Types;
+using mptcc = Mediapipe.Tasks.Components.Containers;
 
 namespace Mediapipe.Unity
 {
@@ -97,6 +98,17 @@ public void Draw(IReadOnlyList<mplt.RelativeKeypoint> targets, float threshold =
       }
     }
 
+    public void Draw(IReadOnlyList<mptcc.NormalizedKeypoint> targets, float threshold = 0.0f)
+    {
+      if (ActivateFor(targets))
+      {
+        CallActionForAll(targets, (annotation, target) =>
+        {
+          if (annotation != null) { annotation.Draw(target, threshold); }
+        });
+      }
+    }
+
     protected override PointAnnotation InstantiateChild(bool isActive = true)
     {
       var annotation = base.InstantiateChild(isActive);

Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/CoordinateSystem/ImageCoordinate.cs

@@ -7,6 +7,7 @@
 using UnityEngine;
 
 using mplt = Mediapipe.LocationData.Types;
+using mptcc = Mediapipe.Tasks.Components.Containers;
 
 namespace Mediapipe.Unity.CoordinateSystem
 {
@@ -476,6 +477,21 @@ public static Vector2 GetPoint(this UnityEngine.Rect rectangle, mplt.RelativeKey
       return ImageNormalizedToPoint(rectangle, relativeKeypoint.X, relativeKeypoint.Y, imageRotation, isMirrored);
     }
 
+    /// <summary>
+    ///   Get the coordinates represented by <paramref name="normalizedKeypoint" /> in the local coordinate system.
+    /// </summary>
+    /// <param name="rectangle">Rectangle to get a point inside</param>
+    /// <param name="imageRotation">
+    ///   Counterclockwise rotation angle of the input image in the image coordinate system.
+    ///   In the local coordinate system, this value will often represent a clockwise rotation angle.
+    /// </param>
+    /// <param name="isMirrored">Set to true if the original coordinates is mirrored</param>
+    public static Vector2 GetPoint(this UnityEngine.Rect rectangle, mptcc.NormalizedKeypoint normalizedKeypoint,
+                                   RotationAngle imageRotation = RotationAngle.Rotation0, bool isMirrored = false)
+    {
+      return ImageNormalizedToPoint(rectangle, normalizedKeypoint.x, normalizedKeypoint.y, imageRotation, isMirrored);
+    }
+
     /// <summary>
     ///   Get the coordinates represented by <paramref name="normalizedLandmark" /> in the local coordinate system.
     /// </summary>
@@ -542,6 +558,41 @@ public static Vector3[] GetRectVertices(this UnityEngine.Rect rectangle, mplt.Re
       return ImageNormalizedToRectVertices(rectangle, boundingBox.Xmin, boundingBox.Ymin, boundingBox.Width, boundingBox.Height, imageRotation, isMirrored);
     }
 
+    /// <summary>
+    ///   Get a Vector3 array which represents <paramref name="rect" />'s vertex coordinates in the local coordinate system.
+    ///   They are ordered clockwise from bottom-left point.
+    /// </summary>
+    /// <param name="rectangle">Rectangle to get a point inside</param>
+    /// <param name="imageWidth">Image width in pixels</param>
+    /// <param name="imageHeight">Image width in pixels</param>
+    /// <param name="imageRotation">
+    ///   Counterclockwise rotation angle of the input image in the image coordinate system.
+    ///   In the local coordinate system, this value will often represent a clockwise rotation angle.
+    /// </param>
+    /// <param name="isMirrored">Set to true if the original coordinates is mirrored</param>
+    public static Vector3[] GetRectVertices(this UnityEngine.Rect rectangle, mptcc.Rect rect, int imageWidth, int imageHeight,
+                                            RotationAngle imageRotation = RotationAngle.Rotation0, bool isMirrored = false)
+    {
+      return ImageToRectVertices(rectangle, rect.left, rect.top, rect.right - rect.left, rect.bottom - rect.top, imageWidth, imageHeight, imageRotation, isMirrored);
+    }
+
+    /// <summary>
+    ///   Get a Vector3 array which represents <paramref name="rect" />'s vertex coordinates in the local coordinate system.
+    ///   They are ordered clockwise from bottom-left point.
+    /// </summary>
+    /// <param name="rectangle">Rectangle to get a point inside</param>
+    /// <param name="imageSize">Image size in pixels</param>
+    /// <param name="imageRotation">
+    ///   Counterclockwise rotation angle of the input image in the image coordinate system.
+    ///   In the local coordinate system, this value will often represent a clockwise rotation angle.
+    /// </param>
+    /// <param name="isMirrored">Set to true if the original coordinates is mirrored</param>
+    public static Vector3[] GetRectVertices(this UnityEngine.Rect rectangle, mptcc.Rect rect, Vector2Int imageSize,
+                                            RotationAngle imageRotation = RotationAngle.Rotation0, bool isMirrored = false)
+    {
+      return ImageToRectVertices(rectangle, rect.left, rect.top, rect.right - rect.left, rect.bottom - rect.top, imageSize, imageRotation, isMirrored);
+    }
+
     /// <summary>
     ///   Get a Vector3 array which represents <paramref name="rect" />'s vertex coordinates in the local coordinate system.
     ///   They are ordered clockwise from bottom-left point.