Add support for media

src/BaileysResponder.ts

@@ -1,5 +1,5 @@
-import type { WAConnection, WAMessage } from '@adiwajshing/baileys'
-import { LanguageProcessor, WAResponderParameters } from "./types";
+import type { MessageOptions, MessageType, WAConnection, WAMessage } from '@adiwajshing/baileys'
+import { Answer, LanguageProcessor, WAResponderParameters } from "./types";
 import { onChatUpdate, onWAMessage } from "./WAResponder";
 import { promises as fs } from "fs";
 
@@ -16,7 +16,37 @@ export const createBaileysResponder = (
 	const connection: WAConnection = new Baileys.WAConnection()
 	connection.autoReconnect = Baileys.ReconnectMode.onAllErrors
 
-	const sendMessage = async(jid: string, message: string, quoted: WAMessage) => {
+	const sendMessage = async(jid: string, answer: Answer, quoted: WAMessage) => {
+
+		let type: MessageType
+		let content: any
+		let options: Partial<MessageOptions> = {}
+		if(typeof answer === 'object' && 'template' in answer) {
+			throw new Error('Baileys responder does not support templates')
+		} else if(typeof answer === 'string') {
+			content = answer
+			type = Baileys.MessageType.conversation
+		} else {
+			if(answer.image) {
+				content = answer.image
+				options = { caption: answer.text }
+				type = Baileys.MessageType.imageMessage
+			} else if(answer.video) {
+				content = answer.video
+				options = { caption: answer.text }
+				type = Baileys.MessageType.videoMessage
+			} else if(answer.audio) {
+				content = answer.video
+				type = Baileys.MessageType.audioMessage
+			} else if(answer.document) {
+				content = answer.document
+				type = Baileys.MessageType.documentMessage
+			} else {
+				content = answer.text
+				type = Baileys.MessageType.conversation
+			}
+		}
+
 		await connection.updatePresence(jid, Baileys.Presence.available)
 		Baileys.delay (250)
 
@@ -26,7 +56,7 @@ export const createBaileysResponder = (
 		await connection.updatePresence(jid, Baileys.Presence.composing)
 		Baileys.delay (2000)
 
-		await connection.sendMessage(jid, message, Baileys.MessageType.text, { quoted })
+		await connection.sendMessage(jid, content, type, { quoted, ...options })
 	}
 
 	connection.on('chat-update', update => (

src/LanguageProcessor.CMDLine.ts → src/CMDLineChat.ts

@@ -1,6 +1,8 @@
+import { randomBytes } from 'crypto'
 import {createInterface} from 'readline'
+import { LanguageProcessor } from './types'
 
-export const chat = (output: (input: string) => Promise<string> | string) => {
+export const chat = ({output}: LanguageProcessor) => {
 	console.log("type 'q' to exit;")
 	const readline = createInterface({input: process.stdin, output: process.stdout})
 
@@ -11,8 +13,12 @@ export const chat = (output: (input: string) => Promise<string> | string) => {
 				process.exit(0)
 			} else {
 				try {
-					const str = await output(ques)
-					console.log("response:\n" + str) 
+					const ctx = { 
+						userId: 'test', 
+						messageId: randomBytes(8).toString('hex') 
+					}
+					const response = await output(ques, ctx)
+					console.log("response:\n", response) 
 				} catch(error) {
 					console.log(`fallback:\n${error.message}\ntrace: ${error.stack}`) 
 				} finally {

src/LanguageProcessor.ts

@@ -1,9 +1,10 @@
-import { randomBytes } from 'crypto'
 import natural from 'natural'
-import { chat as cmdLineChat } from './LanguageProcessor.CMDLine'
-import { InputContext, IntentData, LanguageProcessorMetadata } from './types'
+import { chat as cmdLineChat } from './CMDLineChat'
+import { Answer, InputContext, IntentData, LanguageProcessorMetadata } from './types'
 import { parseTemplate } from './utils'
 
+const OBJECT_KEYS = new Set([ 'template', 'image', 'video', 'audio', 'document' ])
+
 export const createLanguageProcessor = (intents: IntentData[], metadata: LanguageProcessorMetadata = {}) => {
 	const tokenizer = new natural.RegexpTokenizer ({pattern: /\ /})
 	const trie = new natural.Trie(false)
@@ -94,7 +95,7 @@ export const createLanguageProcessor = (intents: IntentData[], metadata: Languag
         return extractedIntents
     }
     const computeOutput = async(data: IntentData, entities: string[], ctx: InputContext) => {    
-		let answer: string | string[]
+		let answer: Answer | Answer[]
         if (typeof data.answer === 'function') { // if the intent requires a function to answer
             answer = await data.answer(entities, ctx)
 		} else if(entities.length === 0) {
@@ -130,17 +131,18 @@ export const createLanguageProcessor = (intents: IntentData[], metadata: Languag
      * @returns the response
      */
     const output = async (input: string, ctx: InputContext) => {
-        const compileAnswer = (strings: string[]) => (
+        const compileAnswer = (strings: (string | { text: string })[]) => (
 			strings.length===1 ? 
-			strings[0] : 
-			strings.map ((str, i) => `*${i+1}.* ${str}`).join("\n")
-		)
+			//@ts-ignore
+			(strings[0].text || strings[0]) : 
+			//@ts-ignore
+			strings.map ((str, i) => `*${i+1}.* ${str.text || str}`).join("\n")
+		) as string
 
         let extractedIntents = extractIntentsAndOptions(input)
-        const intentCount = extractedIntents.length
         if (extractedIntents.length > 1) { // if more than one intent was recognized & a greeting was detected too
             extractedIntents = extractedIntents.filter(intent => !intent.intent.isGreeting)
-        } if (intentCount == 0) {
+        } if (extractedIntents.length === 0) {
             throw new Error( 
 				parseTemplate(metadata.parsingFailedText, { input })
             )
@@ -150,30 +152,30 @@ export const createLanguageProcessor = (intents: IntentData[], metadata: Languag
 			computeOutput(intent, entities, ctx)
 		))
         const outputs = await Promise.allSettled(tasks)
-        const correctOutputs = outputs.filter(output => output.status === 'fulfilled')
+        const correctOutputs = outputs.map(output => (
+			output.status === 'fulfilled' && output.value
+		)).filter(Boolean).flat()
+		const errorOutputs = outputs.map(output => (
+			output.status === 'rejected' && (output.reason?.message || output.reason) as string
+		)).filter(Boolean).flat()
 
-        if (correctOutputs.length > 0) {
-            const strings = correctOutputs.map(output => (output as PromiseFulfilledResult<string>).value).flat()
-            return compileAnswer(strings)
+        if (!!correctOutputs.length) {
+			// check if all answers are strings
+			const allStrings = !correctOutputs.find(item => (
+				typeof item === 'object' && 
+				!!Object.keys(item).filter(k => OBJECT_KEYS.has(k)).length
+			))
+			if(allStrings) {
+				return [
+					compileAnswer(correctOutputs as { text: string }[])
+				]
+			}
+            return correctOutputs
         } else {
-			//@ts-ignore
-			console.log(outputs.map(item => item.reason?.stack))
-            const strings = outputs.map(output => (
-				//@ts-ignore
-				output.value || output.reason
-			)).flat()
-            throw new Error( compileAnswer(strings) )
+            throw new Error(compileAnswer(errorOutputs))
         }
     }
-	const chat = () => cmdLineChat(ip => (
-		output(
-			ip, 
-			{ 
-				userId: 'test', 
-				messageId: randomBytes(8).toString('hex') 
-			}
-		)
-	))
+	const chat = () => cmdLineChat({ output })
 
 	if(!metadata.entityRequiredText) {
 		metadata.entityRequiredText = availableEntities => (

src/SendMammyResponder.ts

@@ -1,6 +1,6 @@
 import type { WAMessage } from "@adiwajshing/baileys"
 import type { AuthenticationController } from "@chatdaddy/authentication-utils"
-import { LanguageProcessor, WAResponderParameters } from "./types"
+import { Answer, LanguageProcessor, WAResponderParameters } from "./types"
 import { onWAMessage } from "./WAResponder"
 import got from "got"
 import { URL } from "url"
@@ -25,14 +25,26 @@ export const createSendMammyResponder = (processor: LanguageProcessor, metadata:
 		const authToken = event.headers['Authorization']?.replace('Bearer ', '')
 		const user = await authController.authenticate(authToken)
 
-		const sendMessage = async(jid: string, text: string, quoted?: WAMessage) => {
+		const sendMessage = async(jid: string, answer: Answer, quoted?: WAMessage) => {
 			const token = await authController.getToken(user.teamId)
 			const timestamp = Math.floor(Date.now()/1000)
+
+			let path = `messages/${jid}`
+			let body: any = {}
+			if(typeof answer === 'object' && 'template' in answer) {
+				path = `${path}/${answer.template}`
+				body = { parameters: answer.parameters }
+			} else if(typeof answer === 'string') {
+				body = { text: answer }
+			} else {
+				body = answer
+			}
+
 			const result = await got.post(
-				new URL(`messages/${jid}`, sendMammyUrl),
+				new URL(path, sendMammyUrl),
 				{
 					body: JSON.stringify({
-						text,
+						...body,
 						scheduleAt: timestamp, // send message now
 						quotedID: quoted?.key.id, // quote the message
 						withTyping: true, // send with typing indicator

src/WAResponder.ts

@@ -1,10 +1,10 @@
-import { LanguageProcessor, WAResponderParameters } from "./types"
-import type { WAMessage, WAChatUpdate } from '@adiwajshing/baileys'
+import { Answer, InputContext, LanguageProcessor, WAResponderParameters } from "./types"
+import type { WAMessage, WAChatUpdate, proto } from '@adiwajshing/baileys'
 
 // file contains generic code to build a WA responder
 
 export type WAMessageParameters = {
-	sendMessage: (jid: string, text: string, quoting?: WAMessage) => Promise<void>
+	sendMessage: (jid: string, reply: Answer, quoting?: WAMessage) => Promise<void>
 	metadata: WAResponderParameters,
 	processor: LanguageProcessor
 }
@@ -46,27 +46,46 @@ export const onWAMessage = async(
 		// get timestamp of message
 		//@ts-ignore
 		const sendTime = message.messageTimestamp?.low || message.messageTimestamp
-		const diff = (new Date().getTime()/1000)-sendTime
+		const diff = (Date.now()/1000)-sendTime
 		if (diff > metadata.minimumDelayTriggerS) {
 			console.log (`been ${diff} seconds since message, responding with delay message to ${senderID}`)
 			// respond if not a group
 			if (!senderID.includes("@g.us")) await sendMessage(senderID, metadata.delayMessage)
 		}
 	}
 
-	let response
+	let responses: Answer[]
+	const ctx: InputContext = {
+		userId: senderID, 
+		messageId: message.key.id 
+	}
+	const [messageContent] = Object.values(message.message)
+	if(typeof messageContent === 'object' && messageContent.contextInfo) {
+		const contextInfo = messageContent.contextInfo as proto.IContextInfo
+		if(contextInfo.remoteJid && contextInfo.stanzaId) {
+			ctx.quotedMessage = {
+				id: contextInfo.stanzaId,
+				remoteJid: contextInfo.remoteJid
+			}
+		}
+	}
 	try {
-		response = await processor.output(
-			messageText, 
-			{  userId: senderID, messageId: message.key.id }
-		)
+		responses = await processor.output(messageText, ctx)
 	} catch (err) {
 		// do not respond if its a group
 		if (senderID.includes("@g.us")) return
-		response = (err.message || err).replace('Error: ', '')
+		responses = [
+			(err.message || err).replace('Error: ', '')
+		]
 	}
-	if (response) {
-		console.log(senderID + " sent message '" + messageText + "', responding with " + response)
-		await sendMessage(senderID, response, message)
+	if (responses) {
+		for(const response of responses) {
+			console.log({
+				message: 'replying',
+				context: ctx,
+				reply: response,
+			})
+			await sendMessage(ctx.userId, response, message)
+		}
 	}
 }

src/example/intents/img-access.ts

@@ -0,0 +1,30 @@
+import { IntentData } from "../../types";
+
+const IMGS = [
+	'https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcSS815rKFCQQ7zkA8mDW5gH45sBT0Eiu7McHw&usqp=CAU',
+	'https://i.pinimg.com/originals/05/1b/7d/051b7d93394fc94c082f1801bc4ccfb2.jpg'
+]
+// sends a random image
+export default {
+	keywords: [
+		'image',
+		'img',
+		'imag'
+	],
+	entities: { },
+	answer: () => {
+		const url = IMGS[Math.floor(Math.random()*IMGS.length)]
+		return {
+			text: 'here is a random image for you',
+			image: { url }
+		}
+	},
+	meta: {
+		userFacingName: [ 'random image' ],
+		description: 'Replies with a random image',
+		examples: [
+			'image pls',
+			'send a random image'
+		]
+	}
+} as IntentData

src/example/intents/index.ts

@@ -1,18 +1,20 @@
 import greeting from './greeting.json'
 import timings from './timings.json'
+import docAccess from './doc-access'
+import imgAccess from './img-access'
 import help from './help'
 
 import { createLanguageProcessor } from '../../LanguageProcessor'
-import docAccess from './doc-access'
 
 export default createLanguageProcessor(
 	[ 
 		greeting, 
 		timings, 
 		docAccess,
-		help([greeting, timings]) // generate help for our two intents
+		imgAccess,
+		help([greeting, timings, docAccess, imgAccess]) // generate help for our intents
 	], 
 	{ 
-		parsingFailedText: "Sorry we couldn't understand {{input}}"
+		parsingFailedText: "Sorry we couldn't understand '{{input}}'"
 	}
 )

src/types.ts

@@ -1,9 +1,27 @@
 export type InputContext = {
+	/** the id of the user */
 	userId: string
+	/** id of the message sent */
 	messageId: string
+	/** info about the message that was quoted (if any) */
+	quotedMessage?: {
+		id: string
+		remoteJid: string
+	} 
+}
+export type FileAnswer = { url: string }
+export type Answer = string | { 
+	text?: string
+	image?: FileAnswer
+	video?: FileAnswer
+	audio?: FileAnswer
+	document?: FileAnswer
+} | { 
+	template: string, 
+	parameters: { [_: string]: any } 
 }
 
-export type IntentAnswer = string | ((entities: string[], ctx: InputContext) => Promise<string> | string)
+export type IntentAnswer = string | ((entities: string[], ctx: InputContext) => Promise<Answer> | Answer)
 export type IntentEntities = {
 	[_: string]: IntentAnswer | { alternates?: string[], value: IntentAnswer }
 }
@@ -35,7 +53,7 @@ export type LanguageProcessorMetadata = {
 	entityRequiredText?: (availableEntities: string[]) => string
 }
 export type LanguageProcessor = {
-	output: (input: string, ctx: InputContext) => Promise<string>
+	output: (input: string, ctx: InputContext) => Promise<Answer[]>
 }
 export type Responser = {
 	start: () => void | Promise<void>