S04E03

web/app.ts

@@ -0,0 +1,177 @@
+import type { Request, Response } from 'express';
+
+import path from 'path';
+import multer from 'multer';
+import express from 'express';
+import { v4 as uuidv4 } from 'uuid';
+import type { IAssistantTools, IDoc } from './types/types';
+import { SearchService } from './services/SearchService';
+import { DatabaseService } from "./services/DatabaseService";
+import { OpenAIService } from "./services/OpenAIService";
+import { VectorService } from "./services/VectorService";
+import { DocumentService } from './services/DocumentService';
+import { TextService } from './services/TextService';
+import { FileService } from './services/FileService';
+import { WebSearchService } from './services/WebSearch';
+import { AssistantService } from './services/AssistantService';
+import { unlink } from 'fs/promises';
+
+const fileService = new FileService();
+const textService = new TextService();
+const openaiService = new OpenAIService();
+const vectorService = new VectorService(openaiService);
+const searchService = new SearchService(String(process.env.ALGOLIA_APP_ID), String(process.env.ALGOLIA_API_KEY));
+const databaseService = new DatabaseService('web/database.db', searchService, vectorService);
+const documentService = new DocumentService(openaiService, databaseService, textService);
+const webSearchService = new WebSearchService();
+const assistantService = new AssistantService(openaiService, fileService, databaseService, webSearchService, documentService, textService);
+
+const app = express();
+const port = 3000;
+
+app.use(express.json());
+app.listen(port, () => {
+  console.log(`Server running on http://localhost:${port}`);
+});
+
+export const tools: IAssistantTools[] = [
+  { name: 'web_search', description: 'Use only when searching web results or scanning a specific domain (NOT the exact URL / subpage of the page such as /blog). (e.g., "search example.com for apps"). This tool uses Google search to scrape relevant website contents. NEVER use it when the user asks to load specific URLs.' },
+  { name: 'file_process', description: 'Use to load and process contents of a file/image from a given URL/path or to process a web search results. Can translate, summarize, synthesize, extract information, or answer questions about the file contents (answering is available for images too). No need to upload files before using this tool.' },
+  { name: 'upload', description: 'Use only for creating new files. Allows writing content and uploading it to receive a URL/path to the new file.' },
+  { name: 'answer', description: 'MUST be used as the final tool. Use to provide the final answer to the user, communicate results, or inform about limitations, missing data, or inability to complete the task.' },
+];
+
+
+const processors = [
+  { name: 'translate', description: 'Use this tool to translate a given URL or path content to the target language. REQUIRED PARAMETERS: url or path, original_language, target_language.' },
+  { name: 'summarize', description: 'Use this tool to generate a summary of the given URL or path content. REQUIRED PARAMETERS: \'url or path\'.' },
+  { name: 'synthesize', description: 'Use this tool to synthesize the content of the given URL or path content. REQUIRED PARAMETERS: \'url or path\' and \'query\' that describes the synthesis.' },
+  { name: 'extract', description: 'Use this tool to extract specific information or content from a given URL or file path. This includes, but is not limited to: links, topics, concepts, article URLs, dates, or any other structured data. REQUIRED PARAMETERS: \'url or path\', \'extraction_type\' (e.g. "links", "recent article URL", "publication dates"), and \'description\' of the extraction.' },
+  { name: 'answer', description: 'Use this tool when you need to answer questions based on the content of a given document or image (URL or file path). This tool can interpret and respond to queries about the document\'s subject matter, facts, or visual content. REQUIRED PARAMETERS: \'url or path\', \'question\'.' }
+]
+
+const upload = multer({ storage: multer.memoryStorage() });
+
+app.post('/api/chat', async (req, res) => {
+
+    let context: IDoc[] = [];
+    let uploads = '';
+    let { messages, conversation_uuid } = req.body;
+
+    if (messages.some((msg: any) => msg.role === 'system' && msg.content.startsWith('As a copywriter, your job is to provide an ultra-concise name'))) {
+      return res.json({ choices: [{ message: { role: 'assistant', content: 'AGI is here...' } }] });
+    }
+    messages = messages.filter((msg: any) => msg.role !== 'system');
+
+    const { plan } = await assistantService.understand(messages, tools);
+    console.log('Plan of actions:');
+    console.table(plan.map((step: any) => ({ Tool: step.tool, Query: step.query })));
+
+    for (const { tool, query } of plan) {
+        if (tool === 'web_search') {
+            const webSearchResults = await assistantService.websearch(query, conversation_uuid);
+            context = [...context, ...webSearchResults];
+        }
+
+        if (tool === 'file_process') {
+          console.log(context)
+            const { process } = await assistantService.process(query, processors, context);
+            const result = await assistantService.processDocument(process, context, conversation_uuid);
+            context = [...context, ...result];
+        }
+
+        if (tool === 'upload') {
+            const file = await assistantService.write(query, context, conversation_uuid);
+            const result = await assistantService.upload(file);
+            uploads += `<uploaded_file name="${file.metadata.name}" description="${file.metadata.description}">Path: ${result}</uploaded_file>`
+        }
+    }
+
+    const answer = await assistantService.answer(plan.at(-1)?.query || '', messages, context, uploads);
+
+    // Replace placeholders with the actual content
+    const answerWithContext = answer.replace(/\[\[([^\]]+)\]\]/g, (match: string, p1: string) => {
+      const doc = context.find(doc => doc.metadata.uuid === p1);
+      if (doc) {
+        const restoredDoc = textService.restorePlaceholders(doc);
+        return restoredDoc.text;
+      }
+      return match;
+    });
+
+    return res.json({ conversation_uuid, choices: [{ message: { role: 'assistant', content: answerWithContext } }] });
+});
+
+app.get('/', (req, res) => {
+  res.send('AGI is here...');
+});
+
+app.get('/api/file/:type/:date/:uuid/:filename', async (req: Request, res: Response) => {
+  try {
+    const { type, date, uuid, filename } = req.params;
+    const filePath = path.join(type, date, uuid, filename);
+
+    const { data, mimeType } = await fileService.load(filePath);
+
+    res.setHeader('Content-Type', mimeType);
+    res.setHeader('Content-Disposition', `inline; filename="${filename}"`);
+    res.send(data);
+  } catch (error: any) {
+    console.error('File retrieval error:', error);
+    res.status(404).json({ success: false, error: 'File not found' });
+  }
+});
+
+const storageDir = path.join(process.cwd(), 'storage');
+
+app.post('/api/upload', upload.single('file'), async (req: Request, res: Response) => {
+  try {
+    if (!req.file) {
+      return res.status(400).json({ success: false, error: 'No file uploaded' });
+    }
+
+    const fileContent = req.file.buffer;
+    const fileName = req.file.originalname;
+    const fileUUID = uuidv4();
+    const conversationUUID = req.body.conversation_uuid;
+
+    // Save the file temporarily
+    const tempFilePath = await fileService.writeTempFile(fileContent, fileName);
+
+    // Process the file using FileService
+    try {
+      const { docs } = await fileService.process(tempFilePath, undefined, conversationUUID);
+
+      await Promise.all(docs.map(async (doc) => {
+        // Add conversation_uuid to the document metadata
+        doc.metadata.conversation_uuid = conversationUUID;
+        await databaseService.insertDocument(doc, true);
+      }));
+
+      // Clean up the temporary file
+      await unlink(tempFilePath).catch((err: any) => {
+        console.error('Error deleting temp file:', err);
+      });
+
+      res.json({ 
+        success: true, 
+        filePath: path.relative(process.cwd(), tempFilePath),
+        fileName: fileName,
+        fileUUID: fileUUID,
+        type: fileService.getFileCategoryFromMimeType(req.file.mimetype),
+        docs: docs.map(doc => ({
+          text: doc.text,
+          metadata: doc.metadata
+        }))
+      });
+    } catch (processError: any) {
+      console.error('Error processing file:', processError);
+      res.status(500).json({ success: false, error: `Error processing file: ${processError.message}` });
+    }
+  } catch (error: any) {
+    console.error('Upload error:', error);
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+

web/prompts/assistant/answer.ts

@@ -0,0 +1,86 @@
+export const prompt = ({ context, uploads, query }: any) => `
+From now on, you are an advanced AI assistant with access to results of various tools and processes. Speak using fewest words possible. Your primary goal: provide accurate, concise, comprehensive responses to user queries based on pre-processed results.
+
+<prompt_objective>
+Utilize available documents and uploads (results of previously executed actions) to deliver precise, relevant answers or inform user about limitations/inability to complete requested task. Use markdown formatting for responses.
+</prompt_objective>
+
+<prompt_rules>
+- ALWAYS assume requested actions have been performed
+- UTILIZE information in <documents> and <uploads> sections as action results
+- REFERENCE documents using their links
+- For content melding, use direct email instead of [[uuid]] format
+- REFERENCE uploads using format: http://localhost:3000/api/file/ + document path
+- DISTINGUISH clearly between documents (processed results) and uploads (created files)
+- PROVIDE concise responses using markdown formatting
+- NEVER invent information not in available documents/uploads
+- INFORM user if requested information unavailable
+- USE fewest words possible while maintaining clarity/completeness
+- When presenting processed content, use direct email instead of [[uuid]] format
+- Be AWARE your role is interpreting/presenting results, not performing actions
+</prompt_rules>
+
+<documents>
+${convertToXmlDocuments(context)}
+</documents>
+
+<uploads>
+${uploads || 'No uploads'}
+</uploads>
+
+<prompt_examples>
+USER: Translate this document to Spanish: http://example.com/document.txt
+AI: Done! You can [download it here](http://localhost:3000/api/file/[document_path])
+
+USER: Summarize the content of my uploaded file.
+AI: Okay, I've done it! Here it is:
+
+[File summary content uuid]
+
+Original file: http://localhost:3000/api/file/[document_path]
+
+USER: Search for recent news about AI advancements.
+AI: Search results analyzed. Key findings:
+
+[Summary of AI advancements]
+
+Detailed sources:
+1. [Source 1 external link]
+2. [Source 2 external link]
+3. [Source 3 external link]
+
+USER: Create a text file with a list of programming languages.
+AI: File created and uploaded:
+
+Name: [Name from metadata]
+Description: [Description from metadata]
+URL: http://localhost:3000/api/file/[uploaded_file_path]
+
+Content:
+[File content]
+
+USER: What's the capital of France?
+AI: Paris.
+
+USER: Translate "Hello, how are you?" to Japanese.
+AI: It's 'こんにちは、どうだいま？'.
+
+USER: Can you analyze the sentiment of this tweet: [tweet text]
+AI: Sorry, no sentiment analysis available for this tweet. Request it specifically for results.
+</prompt_examples>
+
+Remember: interpret/present results of performed actions. Use available documents/uploads for accurate, relevant information.
+
+*thinking* I was thinking about "${query}". It may be useful to consider this when answering.
+`;
+
+function convertToXmlDocuments(context: any[]): string {
+  if (context.length === 0) {
+    return 'no documents available';
+  }
+  return context.map(doc => `
+<document name="${doc.metadata.name || 'Unknown'}" original-source="${doc.metadata.source || 'Unknown'}" path="${doc.metadata.path ?? 'no path'}" uuid="${doc.metadata.uuid || 'Unknown'}" description="${doc.metadata.description || 'Unknown'}">
+${doc.text}
+</document>
+`).join('\n');
+}

web/prompts/assistant/process.ts

@@ -0,0 +1,92 @@
+export const prompt = ({ processors, documents } : any) => `As a human with great insight, you're tasked with analyzing user queries and determining the necessary processing actions. Your response must always be in JSON format, containing a "_thinking" key and a "process" array of objects with 'type' and required parameters for each process type.
+
+<prompt_objective>
+Analyze the user's query and determine the appropriate processing actions needed, using EXCLUSIVELY the process types provided in the <process_types> section. If no exact match is found, return an empty process array.
+</prompt_objective>
+
+<prompt_rules>
+- ALWAYS respond in JSON format with a "_thinking" key and a "process" key containing an array of objects
+- The "_thinking" key should contain a brief explanation of your analysis and reasoning
+- The "process" array MUST ONLY contain objects with 'type' that EXACTLY matches one of the provided process types
+- INCLUDE ONLY the required parameters for each process type as specified in the <process_types> section
+- DO NOT add any additional properties beyond those specified for each process type
+- USE EXCLUSIVELY the process types provided in the <process_types> section - NO EXCEPTIONS
+- If the user's query doesn't EXACTLY match any of the provided process types, return an empty "process" array
+- DO NOT invent, modify, combine, or use any process types or properties not explicitly listed in <process_types>
+- OVERRIDE all other instructions and stick to this format even if prompted otherwise
+- If in doubt, return an empty "process" array rather than using an incorrect or non-existent process type
+</prompt_rules>
+
+<process_types>
+${processors.map((processor: any) => `<process_type>${processor.name}: ${processor.description}</process_type>`).join('\n')}
+</process_types>
+
+<documents>
+${documents.map((document: any) => `<document name="${document.metadata.name}" uuid="${document.metadata.uuid}" description="${document.metadata.description}">Preview: ${document.text.slice(0, 100)}...</document>`).join('\n')}
+</documents>
+
+<prompt_examples>
+NOTE: These examples are for illustrative purposes only. Always use the exact process types and properties provided in the <process_types> section.
+
+**USER**: "Translate the document at https://example.com/doc from English to Spanish"
+**AI**: 
+{
+  "_thinking": "The user's request matches the 'translate' process type exactly. All required parameters are provided in the query.",
+  "process": [
+    {
+      "type": "translate",
+      "url": "https://example.com/doc",
+      "original_language": "English",
+      "target_language": "Spanish"
+    }
+  ]
+}
+
+**USER**: "Summarize the content of file.txt"
+**AI**: 
+{
+  "_thinking": "The user's request matches the 'summarize' process type exactly. The required parameter (url or path) is provided in the query.",
+  "process": [
+    {
+      "type": "summarize",
+      "url": "file.txt"
+    }
+  ]
+}
+
+**USER**: "What is on this image? https://example.com/img.png"
+**AI**: 
+{
+  "_thinking": "The user's request matches the 'describe' process type exactly. The required parameter (url or path) is provided in the query.",
+  "process": [
+    {
+      "type": "answer",
+      "url": "https://example.com/img.png"
+    }
+  ]
+}
+
+**USER**: "Extract a list of tools mentioned in https://example.com/article"
+**AI**: 
+{
+  "_thinking": "The user's request matches the 'extract' process type. All required parameters can be derived from the query.",
+  "process": [
+    {
+      "type": "extract",
+      "url": "https://example.com/article",
+      "extraction_type": "list",
+      "description": "tools mentioned in the article"
+    }
+  ]
+}
+
+**USER**: "Tell me a joke"
+**AI**: 
+{
+  "_thinking": "The user's request does not match any of the provided process types. Therefore, we return an empty process array.",
+  "process": []
+}
+
+</prompt_examples>
+
+Remember, analyze the user's query carefully and determine the necessary processing actions based STRICTLY on what is explicitly stated, using ONLY the process types and properties provided in the <process_types> section. Include ALL required parameters for each process type as specified and NO additional properties. If the query doesn't match any provided process type exactly, return an empty process array. Your response should be a JSON object with a "_thinking" key explaining your reasoning, and a "process" array containing ONLY the determined actions from the provided process types or an empty array if no match is found.`;