Introduce sink of tracked keypoints

README.md

@@ -813,43 +813,43 @@ A `SpeedyKeypoint` object represents a keypoint.
 
 ###### SpeedyKeypoint.position
 
-`SpeedyKeypoint.position: SpeedyPoint2, read-only`
+`SpeedyKeypoint.position: SpeedyPoint2`
 
 The position of the keypoint in the image.
 
 ###### SpeedyKeypoint.x
 
-`SpeedyKeypoint.x: number, read-only`
+`SpeedyKeypoint.x: number`
 
 The x position of the keypoint in the image. A shortcut to `position.x`.
 
 ###### SpeedyKeypoint.y
 
-`SpeedyKeypoint.y: number, read-only`
+`SpeedyKeypoint.y: number`
 
 The y position of the keypoint in the image. A shortcut to `position.y`.
 
 ###### SpeedyKeypoint.lod
 
-`SpeedyKeypoint.lod: number, read-only`
+`SpeedyKeypoint.lod: number`
 
 The level-of-detail (pyramid level) from which the keypoint was extracted, starting from zero. Defaults to `0.0`.
 
 ###### SpeedyKeypoint.scale
 
-`SpeedyKeypoint.scale: number, read-only`
+`SpeedyKeypoint.scale: number`
 
 The scale of the keypoint. This is equivalent to *2 ^ lod*. Defaults to `1.0`.
 
 ###### SpeedyKeypoint.rotation
 
-`SpeedyKeypoint.rotation: number, read-only`
+`SpeedyKeypoint.rotation: number`
 
 The orientation angle of the keypoint, in radians. Defaults to `0.0`.
 
 ###### SpeedyKeypoint.score
 
-`SpeedyKeypoint.score: number, read-only`
+`SpeedyKeypoint.score: number`
 
 The score is a measure associated with the keypoint. Although different detection methods employ different measurement strategies, the larger the score, the "better" the keypoint is considered to be. The score is always a positive value.
 
@@ -881,6 +881,16 @@ The size of the keypoint descriptor, in bytes.
 
 Returns a string representation of the keypoint descriptor.
 
+##### SpeedyTrackedKeypoint
+
+A `SpeedyTrackedKeypoint` is a `SpeedyKeypoint` with the following additional properties:
+
+###### SpeedyTrackerKeypoint.flow
+
+`SpeedyTrackedKeypoint.flow: SpeedyVector2`
+
+A displacement vector associated with the tracked keypoint.
+
 ##### Speedy.Keypoint.Source()
 
 `Speedy.Keypoint.Source(name?: string): SpeedyPipelineNodeKeypointSource`
@@ -914,6 +924,23 @@ Creates a sink of keypoints using the specified name. If the name is not specifi
 |-----------|-----------|-------------|
 | `"in"`    | Keypoints | A set of keypoints to be exported from the pipeline. |
 
+##### Speedy.Keypoint.SinkOfTrackedKeypoints()
+
+`Speedy.Keypoint.SinkOfTrackedKeypoints(name?: string): SpeedyPipelineNodeTrackedKeypointSink`
+
+Creates a sink of tracked keypoints using the specified name. If the name is not specified, Speedy will call this node `"keypoints"`. An array of `SpeedyTrackedKeypoint` objects will be exported from the pipeline.
+
+###### Parameters
+
+The same as `SpeedyPipelineNodeKeypointSink`.
+
+###### Ports
+
+| Port name | Data type | Description |
+|-----------|-----------|-------------|
+| `"in"`    | Keypoints | A set of keypoints to be exported from the pipeline. |
+| `"flow"`  | Vector2   | A set of displacement vectors associated with each keypoint. |
+
 ##### Speedy.Keypoint.Clipper()
 
 `Speedy.Keypoint.Clipper(name?: string): SpeedyPipelineNodeKeypointClipper`

demos/optical-flow.html

@@ -140,8 +140,7 @@ <h1>LK optical flow</h1>
 
     const lk = Speedy.Keypoint.Tracker.LK();
     const mixer = Speedy.Keypoint.Mixer();
-    const sink = Speedy.Keypoint.Sink();
-    const vec2sink = Speedy.Vector2.Sink('flow');
+    const sink = Speedy.Keypoint.SinkOfTrackedKeypoints();
 
     imgsrc.media = media;
     harris.quality = 0.10;
@@ -162,15 +161,15 @@ <h1>LK optical flow</h1>
     mixer.output().connectTo(lk.input('previousKeypoints'));
 
     lk.output().connectTo(sink.input());
-    lk.output('flow').connectTo(vec2sink.input());
+    lk.output('flow').connectTo(sink.input('flow'));
 
-    pipeline.init(imgsrc, grey, pyr, harris, kpsrc, clipper, buf, bufpyr, lk, mixer, sink, vec2sink);
+    pipeline.init(imgsrc, grey, pyr, harris, kpsrc, clipper, buf, bufpyr, lk, mixer, sink);
 
     // Main loop
     let detect = false, clear = false;
     (function() {
         const canvas = createCanvas(media.width, media.height, video.title);
-        let keypoints = [], flow = [], frameReady = false;
+        let keypoints = [], frameReady = false;
 
         async function update()
         {
@@ -190,7 +189,7 @@ <h1>LK optical flow</h1>
 
             // clear all keypoints
             if(clear) {
-                keypoints.length = flow.length = userPoints.length = 0;
+                keypoints.length = userPoints.length = 0;
                 clear = false;
             }
 
@@ -206,7 +205,6 @@ <h1>LK optical flow</h1>
             // run the pipeline
             const result = await pipeline.run();
             keypoints = result.keypoints;
-            flow = result.flow;
 
             // repeat
             frameReady = true;
@@ -218,7 +216,7 @@ <h1>LK optical flow</h1>
         {
             if(frameReady) {
                 media.draw(canvas);
-                renderFlowVectors(canvas, keypoints, flow, 3, '#f22');
+                renderFlowVectors(canvas, keypoints, 3, '#f22');
                 renderFeatures(canvas, keypoints, 1, '#f22');
             }
 
@@ -296,17 +294,17 @@ <h1>LK optical flow</h1>
     context.stroke();
 }
 
-function renderFlowVectors(canvas, features, flow, length = 1, color = 'red')
+function renderFlowVectors(canvas, features, length = 1, color = 'red')
 {
     const context = canvas.getContext('2d');
 
     context.beginPath();
-    for(let i = 0; i < flow.length; i++) {
+    for(let i = 0; i < features.length; i++) {
         const feature = features[i];
-        const v = flow[i];
+        const flow = feature.flow;
 
         // draw flow vector
-        context.moveTo(feature.x - v.x * length, feature.y - v.y * length);
+        context.moveTo(feature.x - flow.x * length, feature.y - flow.y * length);
         context.lineTo(feature.x, feature.y);
     }
     context.strokeStyle = color;

src/core/pipeline/factories/keypoint-factory.js

@@ -21,7 +21,7 @@
 
 import { SpeedyNamespace } from '../../speedy-namespace';
 import { SpeedyPipelineNodeKeypointSource } from '../nodes/keypoints/source';
-import { SpeedyPipelineNodeKeypointSink } from '../nodes/keypoints/sink';
+import { SpeedyPipelineNodeKeypointSink, SpeedyPipelineNodeTrackedKeypointSink } from '../nodes/keypoints/sink';
 import { SpeedyPipelineNodeKeypointClipper } from '../nodes/keypoints/clipper';
 import { SpeedyPipelineNodeKeypointBuffer } from '../nodes/keypoints/buffer';
 import { SpeedyPipelineNodeKeypointMixer } from '../nodes/keypoints/mixer';
@@ -190,6 +190,16 @@ export class SpeedyPipelineKeypointFactory extends SpeedyNamespace
         return new SpeedyPipelineNodeKeypointSink(name);
     }
 
+    /**
+     * Create a sink of tracked keypoints
+     * @param {string} [name]
+     * @returns {SpeedyPipelineNodeTrackedKeypointSink}
+     */
+    static SinkOfTrackedKeypoints(name = undefined)
+    {
+        return new SpeedyPipelineNodeTrackedKeypointSink(name);
+    }
+
     /**
      * Keypoint clipper
      * @param {string} [name]

src/core/pipeline/nodes/keypoints/sink.js

@@ -21,7 +21,7 @@
 
 import { SpeedyPipelineNode, SpeedyPipelineSinkNode } from '../../pipeline-node';
 import { SpeedyPipelineNodeKeypointDetector } from './detectors/detector';
-import { SpeedyPipelineMessageType, SpeedyPipelineMessageWithKeypoints } from '../../pipeline-message';
+import { SpeedyPipelineMessageType, SpeedyPipelineMessageWithKeypoints, SpeedyPipelineMessageWith2DVectors } from '../../pipeline-message';
 import { InputPort, OutputPort } from '../../pipeline-portbuilder';
 import { SpeedyGPU } from '../../../../gpu/speedy-gpu';
 import { SpeedyTextureReader } from '../../../../gpu/speedy-texture-reader';
@@ -31,8 +31,9 @@ import { Utils } from '../../../../utils/utils';
 import { ImageFormat } from '../../../../utils/types';
 import { IllegalOperationError, IllegalArgumentError, AbstractMethodError } from '../../../../utils/errors';
 import { SpeedyPromise } from '../../../../utils/speedy-promise';
-import { SpeedyKeypoint } from '../../../speedy-keypoint';
+import { SpeedyKeypoint, SpeedyTrackedKeypoint } from '../../../speedy-keypoint';
 import { SpeedyKeypointDescriptor } from '../../../speedy-keypoint-descriptor';
+import { SpeedyVector2 } from '../../../speedy-vector';
 import {
     MIN_KEYPOINT_SIZE,
     FIX_RESOLUTION,
@@ -350,9 +351,75 @@ export class SpeedyPipelineNodeKeypointSink extends SpeedyPipelineNodeAbstractKe
 
 /**
  * Gets tracked keypoints out of the pipeline
- * @extends {SpeedyPipelineNodeAbstractKeypointSink<SpeedyKeypoint>}
+ * @extends {SpeedyPipelineNodeAbstractKeypointSink<SpeedyTrackedKeypoint>}
  */
 export class SpeedyPipelineNodeTrackedKeypointSink extends SpeedyPipelineNodeAbstractKeypointSink
 {
-    // TODO
+    /**
+     * Constructor
+     * @param {string} [name] name of the node
+     */
+    constructor(name = 'keypoints')
+    {
+        super(name, 2, [
+            InputPort().expects(SpeedyPipelineMessageType.Keypoints).satisfying(
+                ( /** @type {SpeedyPipelineMessageWithKeypoints} */ msg ) =>
+                    msg.extraSize == 0
+            ),
+            InputPort('flow').expects(SpeedyPipelineMessageType.Vector2)
+        ]);
+    }
+
+    /**
+     * Run the specific task of this node
+     * @param {SpeedyGPU} gpu
+     * @returns {void|SpeedyPromise<void>}
+     */
+    _run(gpu)
+    {
+        const { encodedKeypoints, descriptorSize, extraSize, encoderLength } = /** @type {SpeedyPipelineMessageWithKeypoints} */ ( this.input().read() );
+        const { vectors } = /** @type {SpeedyPipelineMessageWith2DVectors} */ ( this.input('flow').read() );
+
+        // allocate extra space
+        const newDescriptorSize = descriptorSize;
+        const newExtraSize = 4; // 1 pixel per flow vector per keypoint
+        const encodedKeypointsWithExtraSpace = this._allocateExtra(gpu, this._tex[0], encodedKeypoints, descriptorSize, extraSize, newDescriptorSize, newExtraSize);
+
+        // attach flow vectors
+        const newEncoderLength = encodedKeypointsWithExtraSpace.width;
+        const newEncodedKeypoints = (gpu.programs.keypoints.transferToExtra
+            .outputs(newEncoderLength, newEncoderLength, this._tex[1])
+        )(vectors, vectors.width, encodedKeypointsWithExtraSpace, newDescriptorSize, newExtraSize, newEncoderLength);
+
+        // done!
+        return this._download(gpu, newEncodedKeypoints, newDescriptorSize, newExtraSize, newEncoderLength);
+    }
+
+    /**
+     * Instantiate a new keypoint
+     * @param {number} x
+     * @param {number} y
+     * @param {number} lod
+     * @param {number} rotation
+     * @param {number} score
+     * @param {Uint8Array} descriptorBytes
+     * @param {Uint8Array} extraBytes
+     * @returns {SpeedyTrackedKeypoint}
+     */
+    _createKeypoint(x, y, lod, rotation, score, descriptorBytes, extraBytes)
+    {
+        const descriptorSize = descriptorBytes.byteLength;
+        const extraSize = extraBytes.byteLength;
+
+        // read descriptor, if any
+        const descriptor = descriptorSize > 0 ? new SpeedyKeypointDescriptor(descriptorBytes) : null;
+
+        // read flow vector
+        const fx = Utils.decodeFloat16((extraBytes[1] << 8) | extraBytes[0]);
+        const fy = Utils.decodeFloat16((extraBytes[3] << 8) | extraBytes[2]);
+        const flow = new SpeedyVector2(fx, fy);
+
+        // create keypoint
+        return new SpeedyTrackedKeypoint(x, y, lod, rotation, score, descriptor, flow);
+    }
 }

src/core/speedy-keypoint.js

@@ -21,6 +21,7 @@
 
 import { SpeedyKeypointDescriptor } from './speedy-keypoint-descriptor';
 import { SpeedyPoint2 } from './speedy-point';
+import { SpeedyVector2 } from './speedy-vector';
 
 /**
  * Represents a keypoint
@@ -134,4 +135,37 @@ export class SpeedyKeypoint
     {
         return this._descriptor;
     }
+}
+
+/**
+ * Represents a tracked keypoint
+ */
+export class SpeedyTrackedKeypoint extends SpeedyKeypoint
+{
+    /**
+     * Constructor
+     * @param {number} x X position
+     * @param {number} y Y position
+     * @param {number} [lod] Level-of-detail
+     * @param {number} [rotation] Rotation in radians
+     * @param {number} [score] Cornerness measure
+     * @param {SpeedyKeypointDescriptor|null} [descriptor] Keypoint descriptor, if any
+     * @param {SpeedyVector2} [flow] flow vector
+     */
+    constructor(x, y, lod = 0.0, rotation = 0.0, score = 0.0, descriptor = null, flow = new SpeedyVector2(0,0))
+    {
+        super(x, y, lod, rotation, score, descriptor);
+
+        /** @type {SpeedyVector2} flow vector */
+        this._flow = flow;
+    }
+
+    /**
+     * Flow vector
+     * @returns {SpeedyVector2}
+     */
+    get flow()
+    {
+        return this._flow;
+    }
 }