Skip to content

Commit

Permalink
Add workflow statistics to team.start() output
Browse files Browse the repository at this point in the history
- Modify team.start() to include comprehensive workflow statistics in its output
- Remove the need for separate getWorkflowStats() call after workflow completion
- Include duration, task count, agent count, iteration count, LLM usage stats, and cost details in the output
- Ensure consistency between workflow result and associated statistics
- Update relevant documentation and examples to reflect new output structure

Resolves kaiban-ai#62
  • Loading branch information
darielnoel committed Sep 6, 2024
1 parent 0063d7d commit 31e6e7a
Show file tree
Hide file tree
Showing 4 changed files with 275 additions and 50 deletions.
46 changes: 42 additions & 4 deletions playground/react/src/AgentsBoardDebugger.jsx
Original file line number Diff line number Diff line change
@@ -1,5 +1,29 @@
import React, { useState, useEffect } from 'react';

const WorkflowStats = ({ stats }) => {
if (!stats) return null;

return (
<div style={{ margin: '20px 0', padding: '10px', border: '1px solid #ccc', borderRadius: '5px' }}>
<h3 style={{ color: '#666', fontSize: '24px' }}>📊 Workflow Statistics</h3>
<p><strong>Team Name:</strong> {stats.teamName}</p>
<p><strong>Duration:</strong> {stats.duration.toFixed(2)} seconds</p>
<p><strong>Tasks:</strong> {stats.taskCount}</p>
<p><strong>Agents:</strong> {stats.agentCount}</p>
<p><strong>Iterations:</strong> {stats.iterationCount}</p>
<h4>LLM Usage:</h4>
<ul>
<li>Input Tokens: {stats.llmUsageStats.inputTokens}</li>
<li>Output Tokens: {stats.llmUsageStats.outputTokens}</li>
<li>API Calls: {stats.llmUsageStats.callsCount}</li>
<li>Errors: {stats.llmUsageStats.callsErrorCount}</li>
<li>Parsing Errors: {stats.llmUsageStats.parsingErrors}</li>
</ul>
<p><strong>Total Cost:</strong> ${stats.costDetails.totalCost.toFixed(4)}</p>
</div>
);
};

const AgentsBoardDebugger = ({team, title=null}) => {

const useTeamStore = team.useStore();
Expand All @@ -13,7 +37,7 @@ const AgentsBoardDebugger = ({team, title=null}) => {
setInputs: state.setInputs,
workflowContext: state.workflowContext,
provideFeedback: state.provideFeedback,
validateTask: state.validateTask
validateTask: state.validateTask,
}));

const [openSystemMessage, setOpenSystemMessage] = useState({});
Expand All @@ -22,6 +46,9 @@ const AgentsBoardDebugger = ({team, title=null}) => {
const [feedbackContent, setFeedbackContent] = useState('');
const [selectedTaskId, setSelectedTaskId] = useState('');

const [workflowStats, setWorkflowStats] = useState(null);



const handleFeedbackSubmit = () => {
if (selectedTaskId && feedbackContent) {
Expand Down Expand Up @@ -60,12 +87,22 @@ const AgentsBoardDebugger = ({team, title=null}) => {
// console.log('Tasks:', tasks);
// }, [tasks]);

const startTeam = () => {
team.start(inputs)
.catch(error => console.log('Error during team workflow execution'));
const startTeam = async () => {
try {
const output = await team.start(inputs);
if (output.status === 'FINISHED') {
setWorkflowStats(output.stats);
} else if (output.status === 'BLOCKED') {
setWorkflowStats(output.stats);
}
} catch (error) {
console.error("Workflow encountered an error:", error);
setWorkflowStatus('ERRORED');
}
};



return (
<div style={{ padding: '20px', fontFamily: 'Arial' }}>
<h1 style={{ color: '#333' }}>Agents Team Debugger</h1>
Expand Down Expand Up @@ -109,6 +146,7 @@ return (
</div>
))}
</div>
{workflowStats && <WorkflowStats stats={workflowStats} />}
<div style={{ marginTop: '20px' }}>
<h2 style={{ color: '#666', fontSize: '30px', cursor: 'pointer' }} onClick={toggleWorkflowContext}>
{openWorkflowContext ? '▼' : '▶'} 🧠 Workflow Context
Expand Down
68 changes: 66 additions & 2 deletions src/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,8 @@ class Team {
unsubscribe();
resolve({
status,
result: state.workflowResult
result: state.workflowResult,
stats: this.getWorkflowStats()
});
break;
case WORKFLOW_STATUS_enum.ERRORED:
Expand All @@ -167,7 +168,8 @@ class Team {
unsubscribe();
resolve({
status,
result: null
result: null,
stats: this.getWorkflowStats()
});
break;
default:
Expand Down Expand Up @@ -324,6 +326,68 @@ class Team {
getTasks() {
return this.store.getState().tasks;
}
/**
* Retrieves the workflow completion statistics.
* This method finds the completion log in the workflow logs and returns the associated statistics.
*
* @returns {Object|null} The workflow completion statistics, or null if no completion log is found.
* @property {number} startTime - The timestamp representing the workflow start time.
* @property {number} endTime - The timestamp representing the workflow end time.
* @property {number} duration - The duration of the workflow in seconds.
* @property {Object} llmUsageStats - Statistics about the language model usage.
* @property {number} llmUsageStats.inputTokens - The number of input tokens used.
* @property {number} llmUsageStats.outputTokens - The number of output tokens generated.
* @property {number} llmUsageStats.callsCount - The number of LLM API calls made.
* @property {number} llmUsageStats.callsErrorCount - The number of failed LLM API calls.
* @property {number} llmUsageStats.parsingErrors - The number of parsing errors encountered.
* @property {number} iterationCount - The number of iterations in the workflow.
* @property {Object} costDetails - Detailed breakdown of costs associated with the workflow.
* @property {number} costDetails.costInputTokens - The cost of input tokens.
* @property {number} costDetails.costOutputTokens - The cost of output tokens.
* @property {number} costDetails.totalCost - The total cost of the workflow.
* @property {string} teamName - The name of the team that executed the workflow.
* @property {number} taskCount - The total number of tasks in the workflow.
* @property {number} agentCount - The number of agents involved in the workflow.
*/
getWorkflowStats() {
const state = this.store.getState();
const logs = state.workflowLogs;

// Find the log entry for when the workflow was marked as finished or blocked
const completionLog = logs.find(log =>
log.logType === "WorkflowStatusUpdate" &&
(log.workflowStatus === "FINISHED" || log.workflowStatus === "BLOCKED")
);

// Check if a completion log exists and return the specified statistics
if (completionLog) {
const {
startTime,
endTime,
duration,
llmUsageStats,
iterationCount,
costDetails,
teamName,
taskCount,
agentCount
} = completionLog.metadata;

return {
startTime,
endTime,
duration,
llmUsageStats,
iterationCount,
costDetails,
teamName,
taskCount,
agentCount
};
} else {
return null;
}
}
}

export { Agent, Task, Team };
29 changes: 20 additions & 9 deletions src/stores/teamStore.js
Original file line number Diff line number Diff line change
Expand Up @@ -155,10 +155,7 @@ const createTeamStore = (initialState = {}) => {
workflowStatus: WORKFLOW_STATUS_enum.FINISHED,
metadata: {
result: deliverableTask ? deliverableTask.result : lastTaskResult,
...stats,
teamName: get().name,
taskCount: tasks.length,
agentCount: get().agents.length
...stats
},
logType: 'WorkflowStatusUpdate'
};
Expand All @@ -178,6 +175,7 @@ const createTeamStore = (initialState = {}) => {
handleWorkflowError: (task, error) => {
// Detailed console error logging
logger.error(`Workflow Error:`, error.message);
const stats = get().getWorkflowStats();
// Prepare the error log with specific workflow context
const newLog = {
task,
Expand All @@ -186,7 +184,8 @@ const createTeamStore = (initialState = {}) => {
logDescription: `Workflow error encountered: ${error.message}`,
workflowStatus: WORKFLOW_STATUS_enum.ERRORED,
metadata: {
error
error,
...stats
},
logType: 'WorkflowStatusUpdate'
};
Expand All @@ -202,18 +201,27 @@ const createTeamStore = (initialState = {}) => {
handleWorkflowBlocked: ({ task, error }) => {
// Detailed console error logging
logger.warn(`WORKFLOW BLOCKED:`, error.message);


// Get current workflow stats
const stats = get().getWorkflowStats();

// Prepare the error log with specific workflow context
const newLog = {
task,
agent: task.agent,
timestamp: Date.now(),
logDescription: `Workflow blocked encountered: ${error.message}`,
logDescription: `Workflow blocked: ${error.message}`,
workflowStatus: WORKFLOW_STATUS_enum.BLOCKED,
metadata: {
error
error: error.message,
...stats,
teamName: get().name,
taskCount: get().tasks.length,
agentCount: get().agents.length
},
logType: 'WorkflowStatusUpdate'
};
};

// Update state with error details and add new log entry
set(state => ({
Expand Down Expand Up @@ -493,7 +501,10 @@ const createTeamStore = (initialState = {}) => {
duration,
llmUsageStats,
iterationCount,
costDetails
costDetails,
taskCount: get().tasks.length,
agentCount: get().agents.length,
teamName: get().name,
};
},
getCleanedState() {
Expand Down
Loading

0 comments on commit 31e6e7a

Please sign in to comment.