diff --git a/playground/react/src/AgentsBoardDebugger.jsx b/playground/react/src/AgentsBoardDebugger.jsx index 188b361..f5fe012 100644 --- a/playground/react/src/AgentsBoardDebugger.jsx +++ b/playground/react/src/AgentsBoardDebugger.jsx @@ -1,5 +1,29 @@ import React, { useState, useEffect } from 'react'; +const WorkflowStats = ({ stats }) => { + if (!stats) return null; + + return ( +
+

📊 Workflow Statistics

+

Team Name: {stats.teamName}

+

Duration: {stats.duration.toFixed(2)} seconds

+

Tasks: {stats.taskCount}

+

Agents: {stats.agentCount}

+

Iterations: {stats.iterationCount}

+

LLM Usage:

+ +

Total Cost: ${stats.costDetails.totalCost.toFixed(4)}

+
+ ); + }; + const AgentsBoardDebugger = ({team, title=null}) => { const useTeamStore = team.useStore(); @@ -13,7 +37,7 @@ const AgentsBoardDebugger = ({team, title=null}) => { setInputs: state.setInputs, workflowContext: state.workflowContext, provideFeedback: state.provideFeedback, - validateTask: state.validateTask + validateTask: state.validateTask, })); const [openSystemMessage, setOpenSystemMessage] = useState({}); @@ -22,6 +46,9 @@ const AgentsBoardDebugger = ({team, title=null}) => { const [feedbackContent, setFeedbackContent] = useState(''); const [selectedTaskId, setSelectedTaskId] = useState(''); + const [workflowStats, setWorkflowStats] = useState(null); + + const handleFeedbackSubmit = () => { if (selectedTaskId && feedbackContent) { @@ -60,12 +87,22 @@ const AgentsBoardDebugger = ({team, title=null}) => { // console.log('Tasks:', tasks); // }, [tasks]); - const startTeam = () => { - team.start(inputs) - .catch(error => console.log('Error during team workflow execution')); + const startTeam = async () => { + try { + const output = await team.start(inputs); + if (output.status === 'FINISHED') { + setWorkflowStats(output.stats); + } else if (output.status === 'BLOCKED') { + setWorkflowStats(output.stats); + } + } catch (error) { + console.error("Workflow encountered an error:", error); + setWorkflowStatus('ERRORED'); + } }; + return (

Agents Team Debugger

@@ -109,6 +146,7 @@ return (
))} + {workflowStats && }

{openWorkflowContext ? '▼' : '▶'} 🧠 Workflow Context diff --git a/src/index.js b/src/index.js index a2eff05..092c550 100644 --- a/src/index.js +++ b/src/index.js @@ -156,7 +156,8 @@ class Team { unsubscribe(); resolve({ status, - result: state.workflowResult + result: state.workflowResult, + stats: this.getWorkflowStats() }); break; case WORKFLOW_STATUS_enum.ERRORED: @@ -167,7 +168,8 @@ class Team { unsubscribe(); resolve({ status, - result: null + result: null, + stats: this.getWorkflowStats() }); break; default: @@ -324,6 +326,68 @@ class Team { getTasks() { return this.store.getState().tasks; } + /** + * Retrieves the workflow completion statistics. + * This method finds the completion log in the workflow logs and returns the associated statistics. + * + * @returns {Object|null} The workflow completion statistics, or null if no completion log is found. + * @property {number} startTime - The timestamp representing the workflow start time. + * @property {number} endTime - The timestamp representing the workflow end time. + * @property {number} duration - The duration of the workflow in seconds. + * @property {Object} llmUsageStats - Statistics about the language model usage. + * @property {number} llmUsageStats.inputTokens - The number of input tokens used. + * @property {number} llmUsageStats.outputTokens - The number of output tokens generated. + * @property {number} llmUsageStats.callsCount - The number of LLM API calls made. + * @property {number} llmUsageStats.callsErrorCount - The number of failed LLM API calls. + * @property {number} llmUsageStats.parsingErrors - The number of parsing errors encountered. + * @property {number} iterationCount - The number of iterations in the workflow. + * @property {Object} costDetails - Detailed breakdown of costs associated with the workflow. + * @property {number} costDetails.costInputTokens - The cost of input tokens. + * @property {number} costDetails.costOutputTokens - The cost of output tokens. + * @property {number} costDetails.totalCost - The total cost of the workflow. + * @property {string} teamName - The name of the team that executed the workflow. + * @property {number} taskCount - The total number of tasks in the workflow. + * @property {number} agentCount - The number of agents involved in the workflow. + */ + getWorkflowStats() { + const state = this.store.getState(); + const logs = state.workflowLogs; + + // Find the log entry for when the workflow was marked as finished or blocked + const completionLog = logs.find(log => + log.logType === "WorkflowStatusUpdate" && + (log.workflowStatus === "FINISHED" || log.workflowStatus === "BLOCKED") + ); + + // Check if a completion log exists and return the specified statistics + if (completionLog) { + const { + startTime, + endTime, + duration, + llmUsageStats, + iterationCount, + costDetails, + teamName, + taskCount, + agentCount + } = completionLog.metadata; + + return { + startTime, + endTime, + duration, + llmUsageStats, + iterationCount, + costDetails, + teamName, + taskCount, + agentCount + }; + } else { + return null; + } + } } export { Agent, Task, Team }; diff --git a/src/stores/teamStore.js b/src/stores/teamStore.js index 0bf5481..2fbbcc3 100644 --- a/src/stores/teamStore.js +++ b/src/stores/teamStore.js @@ -155,10 +155,7 @@ const createTeamStore = (initialState = {}) => { workflowStatus: WORKFLOW_STATUS_enum.FINISHED, metadata: { result: deliverableTask ? deliverableTask.result : lastTaskResult, - ...stats, - teamName: get().name, - taskCount: tasks.length, - agentCount: get().agents.length + ...stats }, logType: 'WorkflowStatusUpdate' }; @@ -178,6 +175,7 @@ const createTeamStore = (initialState = {}) => { handleWorkflowError: (task, error) => { // Detailed console error logging logger.error(`Workflow Error:`, error.message); + const stats = get().getWorkflowStats(); // Prepare the error log with specific workflow context const newLog = { task, @@ -186,7 +184,8 @@ const createTeamStore = (initialState = {}) => { logDescription: `Workflow error encountered: ${error.message}`, workflowStatus: WORKFLOW_STATUS_enum.ERRORED, metadata: { - error + error, + ...stats }, logType: 'WorkflowStatusUpdate' }; @@ -202,18 +201,27 @@ const createTeamStore = (initialState = {}) => { handleWorkflowBlocked: ({ task, error }) => { // Detailed console error logging logger.warn(`WORKFLOW BLOCKED:`, error.message); + + + // Get current workflow stats + const stats = get().getWorkflowStats(); + // Prepare the error log with specific workflow context const newLog = { task, agent: task.agent, timestamp: Date.now(), - logDescription: `Workflow blocked encountered: ${error.message}`, + logDescription: `Workflow blocked: ${error.message}`, workflowStatus: WORKFLOW_STATUS_enum.BLOCKED, metadata: { - error + error: error.message, + ...stats, + teamName: get().name, + taskCount: get().tasks.length, + agentCount: get().agents.length }, logType: 'WorkflowStatusUpdate' - }; + }; // Update state with error details and add new log entry set(state => ({ @@ -493,7 +501,10 @@ const createTeamStore = (initialState = {}) => { duration, llmUsageStats, iterationCount, - costDetails + costDetails, + taskCount: get().tasks.length, + agentCount: get().agents.length, + teamName: get().name, }; }, getCleanedState() { diff --git a/tests/e2e/__snapshots__/productSpecTeam.test.js.snap b/tests/e2e/__snapshots__/productSpecTeam.test.js.snap index 99f74a1..547d5e2 100644 --- a/tests/e2e/__snapshots__/productSpecTeam.test.js.snap +++ b/tests/e2e/__snapshots__/productSpecTeam.test.js.snap @@ -3720,14 +3720,30 @@ This technical specification outlines the requirements and functionalities neces }, "type": "ReactChampionAgent", }, - "logDescription": "Workflow blocked encountered: Task awaiting validation", + "logDescription": "Workflow blocked: Task awaiting validation", "logType": "WorkflowStatusUpdate", "metadata": { - "duration": undefined, - "endTime": undefined, - "error": [Error: Task awaiting validation], + "agentCount": 3, + "costDetails": { + "costInputTokens": 0.0001, + "costOutputTokens": 0.0001, + "totalCost": 0.0002, + }, + "duration": "[REDACTED]", + "endTime": "[REDACTED]", + "error": "Task awaiting validation", "feedback": {}, - "startTime": undefined, + "iterationCount": 1, + "llmUsageStats": { + "callsCount": 1, + "callsErrorCount": 0, + "inputTokens": 648, + "outputTokens": 239, + "parsingErrors": 0, + }, + "startTime": "[REDACTED]", + "taskCount": 3, + "teamName": "Product Specs Team", }, "task": { "agent": { @@ -14830,14 +14846,30 @@ exports[`Product Spec Team Workflows HITL Features Using OpenAI Agents (1) - han }, "type": "ReactChampionAgent", }, - "logDescription": "Workflow blocked encountered: Task awaiting validation", + "logDescription": "Workflow blocked: Task awaiting validation", "logType": "WorkflowStatusUpdate", "metadata": { - "duration": undefined, - "endTime": undefined, - "error": [Error: Task awaiting validation], + "agentCount": 3, + "costDetails": { + "costInputTokens": 0.0001, + "costOutputTokens": 0.0001, + "totalCost": 0.0002, + }, + "duration": "[REDACTED]", + "endTime": "[REDACTED]", + "error": "Task awaiting validation", "feedback": {}, - "startTime": undefined, + "iterationCount": 1, + "llmUsageStats": { + "callsCount": 1, + "callsErrorCount": 0, + "inputTokens": 648, + "outputTokens": 239, + "parsingErrors": 0, + }, + "startTime": "[REDACTED]", + "taskCount": 3, + "teamName": "Product Specs Team", }, "task": { "agent": { @@ -18785,14 +18817,30 @@ The successful implementation of these specifications will enhance the effective }, "type": "ReactChampionAgent", }, - "logDescription": "Workflow blocked encountered: Task awaiting validation", + "logDescription": "Workflow blocked: Task awaiting validation", "logType": "WorkflowStatusUpdate", "metadata": { - "duration": undefined, - "endTime": undefined, - "error": [Error: Task awaiting validation], + "agentCount": 3, + "costDetails": { + "costInputTokens": 0.0001, + "costOutputTokens": 0.0002, + "totalCost": 0.0003, + }, + "duration": "[REDACTED]", + "endTime": "[REDACTED]", + "error": "Task awaiting validation", "feedback": {}, - "startTime": undefined, + "iterationCount": 1, + "llmUsageStats": { + "callsCount": 1, + "callsErrorCount": 0, + "inputTokens": 648, + "outputTokens": 319, + "parsingErrors": 0, + }, + "startTime": "[REDACTED]", + "taskCount": 3, + "teamName": "Product Specs Team", }, "task": { "agent": { @@ -22461,14 +22509,30 @@ The successful implementation of these specifications will enhance the effective }, "type": "ReactChampionAgent", }, - "logDescription": "Workflow blocked encountered: Task awaiting validation", + "logDescription": "Workflow blocked: Task awaiting validation", "logType": "WorkflowStatusUpdate", "metadata": { - "duration": undefined, - "endTime": undefined, - "error": [Error: Task awaiting validation], + "agentCount": 3, + "costDetails": { + "costInputTokens": 0.0001, + "costOutputTokens": 0.0002, + "totalCost": 0.0003, + }, + "duration": "[REDACTED]", + "endTime": "[REDACTED]", + "error": "Task awaiting validation", "feedback": {}, - "startTime": undefined, + "iterationCount": 1, + "llmUsageStats": { + "callsCount": 1, + "callsErrorCount": 0, + "inputTokens": 998, + "outputTokens": 301, + "parsingErrors": 0, + }, + "startTime": "[REDACTED]", + "taskCount": 3, + "teamName": "Product Specs Team", }, "task": { "agent": { @@ -33923,14 +33987,30 @@ exports[`Product Spec Team Workflows HITL Features Using OpenAI Agents (2) - pro }, "type": "ReactChampionAgent", }, - "logDescription": "Workflow blocked encountered: Task awaiting validation", + "logDescription": "Workflow blocked: Task awaiting validation", "logType": "WorkflowStatusUpdate", "metadata": { - "duration": undefined, - "endTime": undefined, - "error": [Error: Task awaiting validation], + "agentCount": 3, + "costDetails": { + "costInputTokens": 0.0001, + "costOutputTokens": 0.0002, + "totalCost": 0.0003, + }, + "duration": "[REDACTED]", + "endTime": "[REDACTED]", + "error": "Task awaiting validation", "feedback": {}, - "startTime": undefined, + "iterationCount": 1, + "llmUsageStats": { + "callsCount": 1, + "callsErrorCount": 0, + "inputTokens": 648, + "outputTokens": 319, + "parsingErrors": 0, + }, + "startTime": "[REDACTED]", + "taskCount": 3, + "teamName": "Product Specs Team", }, "task": { "agent": { @@ -37599,14 +37679,30 @@ exports[`Product Spec Team Workflows HITL Features Using OpenAI Agents (2) - pro }, "type": "ReactChampionAgent", }, - "logDescription": "Workflow blocked encountered: Task awaiting validation", + "logDescription": "Workflow blocked: Task awaiting validation", "logType": "WorkflowStatusUpdate", "metadata": { - "duration": undefined, - "endTime": undefined, - "error": [Error: Task awaiting validation], + "agentCount": 3, + "costDetails": { + "costInputTokens": 0.0001, + "costOutputTokens": 0.0002, + "totalCost": 0.0003, + }, + "duration": "[REDACTED]", + "endTime": "[REDACTED]", + "error": "Task awaiting validation", "feedback": {}, - "startTime": undefined, + "iterationCount": 1, + "llmUsageStats": { + "callsCount": 1, + "callsErrorCount": 0, + "inputTokens": 998, + "outputTokens": 301, + "parsingErrors": 0, + }, + "startTime": "[REDACTED]", + "taskCount": 3, + "teamName": "Product Specs Team", }, "task": { "agent": { @@ -41124,14 +41220,30 @@ exports[`Product Spec Team Workflows HITL Features Using OpenAI Agents (2) - pro }, "type": "ReactChampionAgent", }, - "logDescription": "Workflow blocked encountered: Task awaiting validation", + "logDescription": "Workflow blocked: Task awaiting validation", "logType": "WorkflowStatusUpdate", "metadata": { - "duration": undefined, - "endTime": undefined, - "error": [Error: Task awaiting validation], + "agentCount": 3, + "costDetails": { + "costInputTokens": 0.0001, + "costOutputTokens": 0.0002, + "totalCost": 0.0003, + }, + "duration": "[REDACTED]", + "endTime": "[REDACTED]", + "error": "Task awaiting validation", "feedback": {}, - "startTime": undefined, + "iterationCount": 1, + "llmUsageStats": { + "callsCount": 1, + "callsErrorCount": 0, + "inputTokens": 648, + "outputTokens": 319, + "parsingErrors": 0, + }, + "startTime": "[REDACTED]", + "taskCount": 3, + "teamName": "Product Specs Team", }, "task": { "agent": {