update blazeface

.api-extractor.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json",
   "mainEntryPointFilePath": "types/lib/src/human.d.ts",
-  "bundledPackages": ["@tensorflow/tfjs-core", "@tensorflow/tfjs-converter"],
+  "bundledPackages": ["@tensorflow/tfjs-core", "@tensorflow/tfjs-data", "@tensorflow/tfjs-layers", "@tensorflow/tfjs-converter"],
   "compiler": {
     "skipLibCheck": false
   },

CHANGELOG.md

@@ -9,8 +9,9 @@
 
 ## Changelog
 
-### **HEAD -> main** 2023/02/02 [email protected]
+### **HEAD -> main** 2023/02/03 [email protected]
 
+- add affectnet
 
 ### **3.0.5** 2023/02/02 [email protected]
 

build.js

@@ -121,18 +121,22 @@ async function main() {
 
   // run api-extractor to create typedef rollup
   const extractorConfig = APIExtractor.ExtractorConfig.loadFileAndPrepare('.api-extractor.json');
-  const extractorResult = APIExtractor.Extractor.invoke(extractorConfig, {
-    localBuild: true,
-    showVerboseMessages: false,
-    messageCallback: (msg) => {
-      msg.handled = true;
-      if (msg.logLevel === 'none' || msg.logLevel === 'verbose' || msg.logLevel === 'info') return;
-      if (msg.sourceFilePath?.includes('/node_modules/')) return;
-      // if (apiExtractorIgnoreList.reduce((prev, curr) => prev || msg.messageId.includes(curr), false)) return; // those are external issues outside of human control
-      log.data('API', { level: msg.logLevel, category: msg.category, id: msg.messageId, file: msg.sourceFilePath, line: msg.sourceFileLine, text: msg.text });
-    },
-  });
-  log.state('API-Extractor:', { succeeeded: extractorResult.succeeded, errors: extractorResult.errorCount, warnings: extractorResult.warningCount });
+  try {
+    const extractorResult = APIExtractor.Extractor.invoke(extractorConfig, {
+      localBuild: true,
+      showVerboseMessages: false,
+      messageCallback: (msg) => {
+        msg.handled = true;
+        if (msg.logLevel === 'none' || msg.logLevel === 'verbose' || msg.logLevel === 'info') return;
+        if (msg.sourceFilePath?.includes('/node_modules/')) return;
+        // if (apiExtractorIgnoreList.reduce((prev, curr) => prev || msg.messageId.includes(curr), false)) return; // those are external issues outside of human control
+        log.data('API', { level: msg.logLevel, category: msg.category, id: msg.messageId, file: msg.sourceFilePath, line: msg.sourceFileLine, text: msg.text });
+      },
+    });
+    log.state('API-Extractor:', { succeeeded: extractorResult.succeeded, errors: extractorResult.errorCount, warnings: extractorResult.warningCount });
+  } catch (err) {
+    log.error('API-Extractor:', err);
+  }
   regExFile('types/human.d.ts', regEx);
   writeFile('export * from \'../types/human\';', 'dist/human.esm-nobundle.d.ts');
   writeFile('export * from \'../types/human\';', 'dist/human.esm.d.ts');

demo/typescript/index.ts

@@ -18,6 +18,9 @@ const humanConfig: Partial<H.Config> = { // user configuration for human, used t
   face: { enabled: true, detector: { rotation: true }, mesh: { enabled: true }, attention: { enabled: false }, iris: { enabled: true }, description: { enabled: true }, emotion: { enabled: true }, antispoof: { enabled: true }, liveness: { enabled: true } },
   body: { enabled: true },
   hand: { enabled: true },
+  // face: { enabled: true, mesh: { enabled: false } },
+  // body: { enabled: false },
+  // hand: { enabled: false },
   object: { enabled: false },
   segmentation: { enabled: false },
   gesture: { enabled: true },

dist/human.esm.js

@@ -38308,10 +38308,10 @@ var getRawBox = (box, input) => box ? [
   (box.endPoint[0] - box.startPoint[0]) / (input.shape[2] || 0),
   (box.endPoint[1] - box.startPoint[1]) / (input.shape[1] || 0)
 ] : [0, 0, 0, 0];
-var scaleBoxCoordinates = (box, factor) => {
+var scaleBoxCoordinates = (box, factor, anchor) => {
   const startPoint = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]];
   const endPoint = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]];
-  const landmarks = box.landmarks;
+  const landmarks = box.landmarks.map((pt) => [(pt[0] + anchor[0]) * factor[0], (pt[1] + anchor[1]) * factor[1]]);
   return { startPoint, endPoint, landmarks, confidence: box.confidence };
 };
 var cutAndResize = (box, image, cropSize) => {
@@ -38558,7 +38558,9 @@ async function getBoxes(inputImage, config3) {
         landmarks: await b.landmarks.array(),
         confidence
       };
-      const scaledBox = scaleBoxCoordinates(rawBox, [(inputImage.shape[2] || 0) / inputSize4, (inputImage.shape[1] || 0) / inputSize4]);
+      b.anchor = Ke(anchors, [nms[i], 0], [1, 2]);
+      const anchor = await b.anchor.data();
+      const scaledBox = scaleBoxCoordinates(rawBox, [(inputImage.shape[2] || 0) / inputSize4, (inputImage.shape[1] || 0) / inputSize4], anchor);
       const enlargedBox = enlargeBox(scaledBox, config3.face["scale"] || faceBoxScaleFactor);
       const squaredBox = squarifyBox(enlargedBox);
       if (squaredBox.size[0] > (((_e = config3.face.detector) == null ? void 0 : _e["minSize"]) || 0) && squaredBox.size[1] > (((_f2 = config3.face.detector) == null ? void 0 : _f2["minSize"]) || 0))
@@ -38787,10 +38789,7 @@ async function predict4(input, config3) {
       face4.boxRaw = getRawBox(box, input);
       face4.score = face4.boxScore;
       face4.size = box.size;
-      face4.mesh = box.landmarks.map((pt) => [
-        (box.startPoint[0] + box.endPoint[0]) / 2 + pt[0] * input.shape[2] / size(),
-        (box.startPoint[1] + box.endPoint[1]) / 2 + pt[1] * input.shape[1] / size()
-      ]);
+      face4.mesh = box.landmarks;
       face4.meshRaw = face4.mesh.map((pt) => [pt[0] / (input.shape[2] || 0), pt[1] / (input.shape[1] || 0), (pt[2] || 0) / size2]);
       for (const key of Object.keys(blazeFaceLandmarks))
         face4.annotations[key] = [face4.mesh[blazeFaceLandmarks[key]]];
@@ -38809,14 +38808,11 @@ async function predict4(input, config3) {
       face4.faceScore = Math.round(100 * faceConfidence[0]) / 100;
       if (face4.faceScore < (((_g2 = config3.face.detector) == null ? void 0 : _g2.minConfidence) || 1)) {
         box.confidence = face4.faceScore;
-        if (config3.face.mesh.keepInvalid) {
+        if (config3.face.mesh["keepInvalid"]) {
           face4.box = clampBox(box, input);
           face4.boxRaw = getRawBox(box, input);
           face4.score = face4.boxScore;
-          face4.mesh = box.landmarks.map((pt) => [
-            (box.startPoint[0] + box.endPoint[0]) / 2 + (box.endPoint[0] + box.startPoint[0]) * pt[0] / size(),
-            (box.startPoint[1] + box.endPoint[1]) / 2 + (box.endPoint[1] + box.startPoint[1]) * pt[1] / size()
-          ]);
+          face4.mesh = box.landmarks;
           face4.meshRaw = face4.mesh.map((pt) => [pt[0] / (input.shape[2] || 1), pt[1] / (input.shape[1] || 1), (pt[2] || 0) / size2]);
           for (const key of Object.keys(blazeFaceLandmarks)) {
             face4.annotations[key] = [face4.mesh[blazeFaceLandmarks[key]]];

package.json

@@ -80,24 +80,26 @@
   "devDependencies": {
     "@html-eslint/eslint-plugin": "^0.16.0",
     "@html-eslint/parser": "^0.16.0",
-    "@microsoft/api-extractor": "^7.34.2",
+    "@microsoft/api-extractor": "^7.34.3",
     "@tensorflow/tfjs-backend-cpu": "^4.2.0",
     "@tensorflow/tfjs-backend-wasm": "^4.2.0",
     "@tensorflow/tfjs-backend-webgl": "^4.2.0",
     "@tensorflow/tfjs-backend-webgpu": "0.0.1-alpha.17",
     "@tensorflow/tfjs-converter": "^4.2.0",
     "@tensorflow/tfjs-core": "^4.2.0",
+    "@tensorflow/tfjs-data": "^4.2.0",
+    "@tensorflow/tfjs-layers": "^4.2.0",
     "@tensorflow/tfjs-node": "^4.2.0",
     "@tensorflow/tfjs-node-gpu": "^4.2.0",
-    "@types/node": "^18.11.18",
+    "@types/node": "^18.11.19",
     "@types/offscreencanvas": "^2019.7.0",
-    "@typescript-eslint/eslint-plugin": "^5.50.0",
-    "@typescript-eslint/parser": "^5.50.0",
-    "@vladmandic/build": "^0.7.15",
+    "@typescript-eslint/eslint-plugin": "^5.51.0",
+    "@typescript-eslint/parser": "^5.51.0",
+    "@vladmandic/build": "0.7.15",
     "@vladmandic/pilogger": "^0.4.7",
     "@vladmandic/tfjs": "github:vladmandic/tfjs",
     "canvas": "^2.11.0",
-    "esbuild": "^0.17.5",
+    "esbuild": "^0.17.6",
     "eslint": "8.33.0",
     "eslint-config-airbnb-base": "^15.0.0",
     "eslint-plugin-html": "^7.1.0",

src/exports.ts

@@ -9,7 +9,7 @@ export * from './result';
 /**
  * Explict reexport of main @tensorflow/tfjs types
  */
-export type { Tensor, Tensor4D, GraphModel, Rank } from './tfjs/types';
+export type { Tensor, Tensor1D, Tensor2D, Tensor3D, Tensor4D, TensorLike, GraphModel, Rank } from './tfjs/types';
 
 // re-export types
 export type { DrawOptions } from './draw/options';

src/face/blazeface.ts

@@ -82,6 +82,7 @@ export async function getBoxes(inputImage: Tensor4D, config: Config): Promise<De
   const scores = await t.scores.data();
   for (let i = 0; i < nms.length; i++) {
     const confidence = scores[nms[i]];
+
     if (confidence > (config.face.detector?.minConfidence || 0)) {
       const b: Record<string, Tensor> = {};
       b.bbox = tf.slice(t.boxes, [nms[i], 0], [1, -1]);
@@ -95,7 +96,9 @@ export async function getBoxes(inputImage: Tensor4D, config: Config): Promise<De
         landmarks: (await b.landmarks.array()) as Point[],
         confidence,
       };
-      const scaledBox = util.scaleBoxCoordinates(rawBox, [(inputImage.shape[2] || 0) / inputSize, (inputImage.shape[1] || 0) / inputSize]);
+      b.anchor = tf.slice(anchors as Tensor, [nms[i], 0], [1, 2]);
+      const anchor = await b.anchor.data();
+      const scaledBox = util.scaleBoxCoordinates(rawBox, [(inputImage.shape[2] || 0) / inputSize, (inputImage.shape[1] || 0) / inputSize], anchor);
       const enlargedBox = util.enlargeBox(scaledBox, config.face['scale'] || faceBoxScaleFactor);
       const squaredBox = util.squarifyBox(enlargedBox);
       if (squaredBox.size[0] > (config.face.detector?.['minSize'] || 0) && squaredBox.size[1] > (config.face.detector?.['minSize'] || 0)) boxes.push(squaredBox);

src/face/facemesh.ts

@@ -78,10 +78,7 @@ export async function predict(input: Tensor4D, config: Config): Promise<FaceResu
       face.boxRaw = util.getRawBox(box, input);
       face.score = face.boxScore;
       face.size = box.size;
-      face.mesh = box.landmarks.map((pt) => [
-        ((box.startPoint[0] + box.endPoint[0]) / 2) + (pt[0] * input.shape[2] / blazeface.size()),
-        ((box.startPoint[1] + box.endPoint[1]) / 2) + (pt[1] * input.shape[1] / blazeface.size()),
-      ]);
+      face.mesh = box.landmarks;
       face.meshRaw = face.mesh.map((pt) => [pt[0] / (input.shape[2] || 0), pt[1] / (input.shape[1] || 0), (pt[2] || 0) / size]);
       for (const key of Object.keys(coords.blazeFaceLandmarks)) face.annotations[key] = [face.mesh[coords.blazeFaceLandmarks[key] as number]]; // add annotations
     } else if (!model) { // mesh enabled, but not loaded
@@ -98,14 +95,11 @@ export async function predict(input: Tensor4D, config: Config): Promise<FaceResu
       face.faceScore = Math.round(100 * faceConfidence[0]) / 100;
       if (face.faceScore < (config.face.detector?.minConfidence || 1)) { // low confidence in detected mesh
         box.confidence = face.faceScore; // reset confidence of cached box
-        if (config.face.mesh.keepInvalid) {
+        if (config.face.mesh['keepInvalid']) {
           face.box = util.clampBox(box, input);
           face.boxRaw = util.getRawBox(box, input);
           face.score = face.boxScore;
-          face.mesh = box.landmarks.map((pt) => [
-            ((box.startPoint[0] + box.endPoint[0])) / 2 + ((box.endPoint[0] + box.startPoint[0]) * pt[0] / blazeface.size()),
-            ((box.startPoint[1] + box.endPoint[1])) / 2 + ((box.endPoint[1] + box.startPoint[1]) * pt[1] / blazeface.size()),
-          ]);
+          face.mesh = box.landmarks;
           face.meshRaw = face.mesh.map((pt) => [pt[0] / (input.shape[2] || 1), pt[1] / (input.shape[1] || 1), (pt[2] || 0) / size]);
           for (const key of Object.keys(coords.blazeFaceLandmarks)) {
             face.annotations[key] = [face.mesh[coords.blazeFaceLandmarks[key] as number]]; // add annotations

src/face/facemeshutil.ts

@@ -31,10 +31,18 @@ export const getRawBox = (box, input): Box => (box ? [
   (box.endPoint[1] - box.startPoint[1]) / (input.shape[1] || 0),
 ] : [0, 0, 0, 0]);
 
-export const scaleBoxCoordinates = (box, factor) => {
+export const scaleBoxCoordinates = (box, factor, anchor) => {
   const startPoint: Point = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]];
   const endPoint: Point = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]];
-  const landmarks = box.landmarks;
+  // const centerPoint = [(startPoint[0] + endPoint[0]) / 2, (startPoint[1] + endPoint[1]) / 2];
+  const landmarks = box.landmarks.map((pt) => [(pt[0] + anchor[0]) * factor[0], (pt[1] + anchor[1]) * factor[1]]);
+  /**
+  face.mesh = box.landmarks.map((pt) => [
+    ((box.startPoint[0] + box.endPoint[0]) / 2) + (pt[0] * input.shape[2] / blazeface.size()),
+    ((box.startPoint[1] + box.endPoint[1]) / 2) + (pt[1] * input.shape[1] / blazeface.size()),
+  ]);
+  */
+
   return { startPoint, endPoint, landmarks, confidence: box.confidence };
 };
 

src/image/enhance.ts

@@ -3,7 +3,7 @@
  */
 
 import * as tf from 'dist/tfjs.esm.js';
-import type { Tensor } from '../exports';
+import type { Tensor } from '../tfjs/types';
 
 export async function histogramEqualization(inputImage: Tensor): Promise<Tensor> {
   const squeeze = inputImage.shape.length === 4 ? tf.squeeze(inputImage) : inputImage;