feat(server): complete telemetry client

package.json

@@ -110,6 +110,7 @@
     "@types/archiver": "5.3.2",
     "@types/cli-spinner": "0.2.1",
     "@types/fluent-ffmpeg": "2.1.20",
+    "@types/getos": "3.0.1",
     "@types/node": "18.7.13",
     "@types/node-wav": "0.0.0",
     "@typescript-eslint/eslint-plugin": "5.55.0",

scripts/check.js

@@ -11,10 +11,10 @@ import axios from 'axios'
 import osName from 'os-name'
 import getos from 'getos'
 
-import { version } from '@@/package.json'
 import { LogHelper } from '@/helpers/log-helper'
 import { SystemHelper } from '@/helpers/system-helper'
 import {
+  LEON_VERSION,
   PYTHON_BRIDGE_BIN_PATH,
   TCP_SERVER_BIN_PATH,
   TCP_SERVER_VERSION,
@@ -124,8 +124,8 @@ dotenv.config()
      */
 
     LogHelper.info('Leon version')
-    LogHelper.success(`${version}\n`)
-    pastebinData.leonVersion = version
+    LogHelper.success(`${LEON_VERSION}\n`)
+    pastebinData.leonVersion = LEON_VERSION
 
     /**
      * Environment checking

scripts/generate/generate-json-schemas.js

@@ -20,8 +20,8 @@ import {
 
 /**
  * Generate JSON schemas
- * @param {string} categoryName 
- * @param {Map<string, Object>} schemas 
+ * @param {string} categoryName
+ * @param {Map<string, Object>} schemas
  */
 export const generateSchemas = async (categoryName, schemas) => {
   const categorySchemasPath = path.join(process.cwd(), 'schemas', categoryName)
@@ -65,10 +65,7 @@ export default async () => {
       'voice-config-schemas',
       new Map([
         ['amazon', amazonVoiceConfiguration],
-        [
-          'google-cloud',
-          googleCloudVoiceConfiguration
-        ],
+        ['google-cloud', googleCloudVoiceConfiguration],
         ['watson-stt', watsonVoiceConfiguration],
         ['watson-tts', watsonVoiceConfiguration]
       ])

scripts/setup/create-instance-id.js

@@ -2,11 +2,11 @@ import fs from 'node:fs'
 
 import { LEON_FILE_PATH } from '@/constants'
 import { Telemetry } from '@/telemetry'
+import { LogHelper } from '@/helpers/log-helper'
 
-// TODO: export default
-;(async () => {
+export default async () => {
   try {
-    const { instanceID, birthDate } = await Telemetry.postinstall()
+    const { instanceID, birthDate } = await Telemetry.postInstall()
 
     if (!fs.existsSync(LEON_FILE_PATH)) {
       await fs.promises.writeFile(
@@ -22,8 +22,8 @@ import { Telemetry } from '@/telemetry'
       )
     }
 
-    // resolve()
+    LogHelper.success(`Instance ID created: ${instanceID}`)
   } catch (e) {
-    // reject(new Error(`Failed to create instance ID: ${e}`))
+    LogHelper.warning(`Failed to create the instance ID: ${e}`)
   }
-})()
+}

scripts/setup/setup.js

@@ -9,7 +9,7 @@ import setupDotenv from './setup-dotenv'
 import setupCore from './setup-core'
 import setupSkillsConfig from './setup-skills-config'
 import setupPythonBinaries from './setup-python-binaries'
-// import createInstanceID from './create-instance-id'
+import createInstanceID from './create-instance-id'
 
 // Do not load ".env" file because it is not created yet
 
@@ -27,7 +27,7 @@ import setupPythonBinaries from './setup-python-binaries'
     await generateJsonSchemas()
     LoaderHelper.start()
     await train()
-    // await createInstanceID()
+    await createInstanceID()
 
     LogHelper.default('')
     LogHelper.success('Hooray! Leon is installed and ready to go!')

server/src/constants.ts

@@ -53,6 +53,8 @@ export const PYTHON_BRIDGE_BIN_PATH = path.join(
   PYTHON_BRIDGE_BIN_NAME
 )
 
+export const LEON_VERSION = process.env['npm_package_version']
+
 /**
  * spaCy models
  * Find new spaCy models: https://github.com/explosion/spacy-models/releases
@@ -96,6 +98,8 @@ export const HAS_LOGGER = process.env['LEON_LOGGER'] === 'true'
 export const TCP_SERVER_HOST = process.env['LEON_PY_TCP_SERVER_HOST']
 export const TCP_SERVER_PORT = Number(process.env['LEON_PY_TCP_SERVER_PORT'])
 
+export const IS_TELEMETRY_ENABLED = process.env['LEON_TELEMETRY'] === 'true'
+
 /**
  * Paths
  */

server/src/core/http-server/api/info/get.ts

@@ -1,7 +1,7 @@
 import type { FastifyPluginAsync } from 'fastify'
 
-import { version } from '@@/package.json'
 import {
+  LEON_VERSION,
   HAS_AFTER_SPEECH,
   HAS_LOGGER,
   HAS_STT,
@@ -39,7 +39,7 @@ export const getInfo: FastifyPluginAsync<APIOptions> = async (
           enabled: HAS_TTS,
           provider: TTS_PROVIDER
         },
-        version
+        version: LEON_VERSION
       })
     }
   })

server/src/core/http-server/http-server.ts

@@ -5,8 +5,12 @@ import fastifyStatic from '@fastify/static'
 import { Type } from '@sinclair/typebox'
 import type { Static } from '@sinclair/typebox'
 
-import { version } from '@@/package.json'
-import { LEON_NODE_ENV, HAS_LOGGER, HAS_OVER_HTTP } from '@/constants'
+import {
+  LEON_VERSION,
+  LEON_NODE_ENV,
+  HAS_LOGGER,
+  HAS_OVER_HTTP
+} from '@/constants'
 import { LogHelper } from '@/helpers/log-helper'
 import { DateHelper } from '@/helpers/date-helper'
 import { corsMidd } from '@/core/http-server/plugins/cors'
@@ -60,7 +64,7 @@ export default class HTTPServer {
 
     LogHelper.title('Initialization')
     LogHelper.info(`The current env is ${LEON_NODE_ENV}`)
-    LogHelper.info(`The current version is ${version}`)
+    LogHelper.info(`The current version is ${LEON_VERSION}`)
 
     LogHelper.info(`The current time zone is ${DateHelper.getTimeZone()}`)
 

server/src/core/nlp/nlu/ner.ts

@@ -156,6 +156,17 @@ export default class NER {
     }
   }
 
+  public anonymizeEntities(
+    utterance: NLPUtterance,
+    entities: NEREntity[]
+  ): NLPUtterance {
+    entities.forEach((entity) => {
+      utterance = utterance.replace(entity.sourceText, `{${entity.entity}}`)
+    })
+
+    return utterance
+  }
+
   /**
    * Get spaCy entities from the TCP server
    */

server/src/core/nlp/nlu/nlu.ts

@@ -21,6 +21,7 @@ import { LangHelper } from '@/helpers/lang-helper'
 import { ActionLoop } from '@/core/nlp/nlu/action-loop'
 import { SlotFilling } from '@/core/nlp/nlu/slot-filling'
 import Conversation, { DEFAULT_ACTIVE_CONTEXT } from '@/core/nlp/conversation'
+import { Telemetry } from '@/telemetry'
 
 type NLUProcessResult = Promise<Partial<
   BrainProcessResult & {
@@ -295,6 +296,17 @@ export default class NLU {
         const processingTimeEnd = Date.now()
         const processingTime = processingTimeEnd - processingTimeStart
 
+        Telemetry.utterance({
+          value: utterance,
+          entities: this.nluResult.entities,
+          triggeredDomain: this.nluResult.classification.domain,
+          triggeredSkill: this.nluResult.classification.skill,
+          triggeredAction: this.nluResult.classification.action,
+          probability: this.nluResult.classification.confidence,
+          language: BRAIN.lang,
+          executionTime: processedData?.executionTime || 0
+        })
+
         return resolve({
           processingTime, // In ms, total time
           ...processedData,

server/src/helpers/log-helper.ts

@@ -4,7 +4,7 @@ import path from 'node:path'
 import { DateHelper } from '@/helpers/date-helper'
 
 export class LogHelper {
-  static readonly ERRORS_PATH = path.join(
+  static readonly ERRORS_FILE_PATH = path.join(
     __dirname,
     '..',
     '..',
@@ -47,10 +47,10 @@ export class LogHelper {
   public static error(value: string): void {
     const data = `${DateHelper.getDateTime()} - ${value}`
 
-    if (fs.existsSync(LogHelper.ERRORS_PATH)) {
-      fs.appendFileSync(LogHelper.ERRORS_PATH, `\n${data}`)
+    if (fs.existsSync(this.ERRORS_FILE_PATH)) {
+      fs.appendFileSync(this.ERRORS_FILE_PATH, `\n${data}`)
     } else {
-      fs.writeFileSync(LogHelper.ERRORS_PATH, data, { flag: 'wx' })
+      fs.writeFileSync(this.ERRORS_FILE_PATH, data, { flag: 'wx' })
     }
 
     console.error('\x1b[31m🚨 %s\x1b[0m', value)
@@ -83,4 +83,25 @@ export class LogHelper {
   public static timeEnd(value: string): void {
     console.timeEnd(`🕑 \x1b[36m${value}\x1b[0m`)
   }
+
+  /**
+   * Parse error logs and return an array of log errors
+   * @example parseErrorLogs() // 'Failed to connect to the TCP server: Error: read ECONNRESET'
+   */
+  public static async parseErrorLogs(): Promise<string[]> {
+    const errorFileContent = await fs.promises.readFile(
+      LogHelper.ERRORS_FILE_PATH,
+      'utf8'
+    )
+    const errorLogs = errorFileContent
+      .trim()
+      .split(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[+-]\d{2}:\d{2} - /)
+
+    // Remove the first empty string if there's one
+    if (errorLogs[0] === '') {
+      errorLogs.shift()
+    }
+
+    return errorLogs
+  }
 }

server/src/helpers/system-helper.ts

@@ -118,4 +118,23 @@ export class SystemHelper {
   public static getTotalRAM(): number {
     return Number((os.totalmem() / (1_024 * 1_024 * 1_024)).toFixed(2))
   }
+
+  /**
+   * Get the Node.js version of the current process
+   * @example getNodeJSVersion() // 'v18.15.0'
+   */
+  public static getNodeJSVersion(): string {
+    return process.version || '0.0.0'
+  }
+
+  /**
+   * Get the npm version used to run the current process
+   * @example getNPMVersion() // '9.5.0'
+   */
+  public static getNPMVersion(): string {
+    return (
+      process.env['npm_config_user_agent']?.split('/')[1]?.split(' ')[0] ||
+      '0.0.0'
+    )
+  }
 }

server/src/index.ts

@@ -1,12 +1,16 @@
 import { spawn } from 'node:child_process'
+import fs from 'node:fs'
 
 import {
   IS_DEVELOPMENT_ENV,
+  IS_TELEMETRY_ENABLED,
   LANG as LEON_LANG,
   TCP_SERVER_BIN_PATH
 } from '@/constants'
 import { TCP_CLIENT, HTTP_SERVER, SOCKET_SERVER } from '@/core'
+import { Telemetry } from '@/telemetry'
 import { LangHelper } from '@/helpers/lang-helper'
+import { LogHelper } from '@/helpers/log-helper'
 ;(async (): Promise<void> => {
   process.title = 'leon'
 
@@ -31,4 +35,39 @@ import { LangHelper } from '@/helpers/lang-helper'
 
   // Start the socket server
   SOCKET_SERVER.init()
+
+  // Telemetry events
+  if (IS_TELEMETRY_ENABLED) {
+    Telemetry.start()
+
+    // Watch for errors in the error log file and report them to the telemetry service
+    fs.watchFile(LogHelper.ERRORS_FILE_PATH, async () => {
+      const logErrors = await LogHelper.parseErrorLogs()
+      const lastError = logErrors[logErrors.length - 1] || ''
+
+      Telemetry.error(lastError)
+    })
+
+    setInterval(() => {
+      Telemetry.heartbeat()
+    }, 1_000 * 3_600)
+    ;[
+      'exit',
+      'SIGINT',
+      'SIGUSR1',
+      'SIGUSR2',
+      'uncaughtException',
+      'SIGTERM'
+    ].forEach((eventType) => {
+      process.on(eventType, () => {
+        Telemetry.stop()
+
+        global.tcpServerProcess.kill()
+
+        setTimeout(() => {
+          process.exit(0)
+        }, 1_000)
+      })
+    })
+  }
 })()

server/src/schemas/voice-config-schemas.ts

@@ -2,39 +2,33 @@ import type { Static } from '@sinclair/typebox'
 import { Type } from '@sinclair/typebox'
 
 export const amazonVoiceConfiguration = Type.Strict(
-  Type.Object(
-    {
-      credentials: Type.Object({
-        accessKeyId: Type.String(),
-        secretAccessKey: Type.String()
-      }),
-      region: Type.String()
-    }
-  )
+  Type.Object({
+    credentials: Type.Object({
+      accessKeyId: Type.String(),
+      secretAccessKey: Type.String()
+    }),
+    region: Type.String()
+  })
 )
 export const googleCloudVoiceConfiguration = Type.Strict(
-  Type.Object(
-    {
-      type: Type.Literal('service_account'),
-      project_id: Type.String(),
-      private_key_id: Type.String(),
-      private_key: Type.String(),
-      client_email: Type.String({ format: 'email' }),
-      client_id: Type.String(),
-      auth_uri: Type.String({ format: 'uri' }),
-      token_uri: Type.String({ format: 'uri' }),
-      auth_provider_x509_cert_url: Type.String({ format: 'uri' }),
-      client_x509_cert_url: Type.String({ format: 'uri' })
-    }
-  )
+  Type.Object({
+    type: Type.Literal('service_account'),
+    project_id: Type.String(),
+    private_key_id: Type.String(),
+    private_key: Type.String(),
+    client_email: Type.String({ format: 'email' }),
+    client_id: Type.String(),
+    auth_uri: Type.String({ format: 'uri' }),
+    token_uri: Type.String({ format: 'uri' }),
+    auth_provider_x509_cert_url: Type.String({ format: 'uri' }),
+    client_x509_cert_url: Type.String({ format: 'uri' })
+  })
 )
 export const watsonVoiceConfiguration = Type.Strict(
-  Type.Object(
-    {
-      apikey: Type.String(),
-      url: Type.String({ format: 'uri' })
-    }
-  )
+  Type.Object({
+    apikey: Type.String(),
+    url: Type.String({ format: 'uri' })
+  })
 )
 
 export type AmazonVoiceConfigurationSchema = Static<