refactor: migrate agent system from XML tool calling to OpenAI SDK with native function calling
- Replace custom XML-based tool parsing (XMLToolExecutor) with OpenAI SDK's native tool_calls via `openai` npm package (dangerouslyAllowBrowser) - Consolidate 4 LLM providers (OpenAI, Claude, Gemini, ClaudeOpenRouter) into a single OpenAI SDK-based LLMProvider compatible with any OpenAI-style API (OpenAI, OpenRouter, Ollama, vLLM) - Move agentic tool execution loop from ChatBox into AgentCore - Update Message type to support tool roles, tool_calls, and tool_call_id - Update system prompt to remove XML formatting instructions (~45% smaller) - Remove AttemptCompletionTool (replaced by stop_reason detection), ThinkTool, and ThinkingTool - Polish tool descriptions for OpenAI function calling schema compliance - Normalize base URLs by stripping /chat/completions suffix
This commit is contained in:
+67
-125
@@ -7,9 +7,8 @@ import { KGCore } from './KGCore';
|
||||
import { KGMidiRegion } from './region/KGMidiRegion';
|
||||
import { convertRegionToABCNotation } from '../util/abcNotationUtil';
|
||||
import { extractXMLFromString } from '../util/xmlUtil';
|
||||
import { XMLToolExecutor } from '../agent/core/XMLToolExecutor';
|
||||
import { AgentCore } from '../agent/core/AgentCore';
|
||||
import { AttemptCompletionTool } from '../agent/tools/AttemptCompletionTool';
|
||||
import { AVAILABLE_TOOLS } from '../agent/tools';
|
||||
import type { TimeSignature } from '../types/projectTypes';
|
||||
import { useProjectStore } from '../stores/projectStore';
|
||||
|
||||
@@ -28,8 +27,7 @@ export class KGDebugger {
|
||||
'debugSelectedItems()',
|
||||
'createTestRegion()',
|
||||
'testExtractXMLFromString(input)',
|
||||
'testXMLToolExecution(input)',
|
||||
'testAttemptCompletion(comment)',
|
||||
'testToolCall(jsonInput)',
|
||||
'inputChatBox(content, interval?)'
|
||||
]);
|
||||
}
|
||||
@@ -264,129 +262,72 @@ export class KGDebugger {
|
||||
}
|
||||
|
||||
/**
|
||||
* Test the complete XML tool execution pipeline
|
||||
* @param input - String containing XML tool invocations to execute
|
||||
* Test native tool calling by executing tool calls from a JSON string.
|
||||
* Accepts a single tool call object or an array of tool call objects.
|
||||
*
|
||||
* Usage examples in browser console:
|
||||
*
|
||||
* // Single tool call:
|
||||
* await KGStudio.KGDebugger.testToolCall('{"name":"read_music","arguments":{"start_beat":0,"length":8}}')
|
||||
*
|
||||
* // Multiple tool calls:
|
||||
* await KGStudio.KGDebugger.testToolCall('[{"name":"remove_notes","arguments":{"start_beat":0,"end_beat":4}},{"name":"add_notes","arguments":{"notes":[{"pitch":"C4","start_beat":0,"length":1}]}}]')
|
||||
*
|
||||
* // Can also pass a JS object directly (no need to stringify):
|
||||
* await KGStudio.KGDebugger.testToolCall({name:"read_music",arguments:{start_beat:0}})
|
||||
*
|
||||
* @param input - JSON string, object, or array of tool call(s).
|
||||
* Each tool call should have: { name: string, arguments: object }
|
||||
*/
|
||||
public async testXMLToolExecution(input: string): Promise<void> {
|
||||
console.log('------------ ASSISTANT ------------');
|
||||
console.log(input);
|
||||
console.log('-----------------------------------');
|
||||
|
||||
public async testToolCall(input: string | Record<string, unknown> | Record<string, unknown>[]): Promise<void> {
|
||||
try {
|
||||
// Extract XML blocks first to get tool names (same logic as ChatBox)
|
||||
const xmlBlocks = extractXMLFromString(input);
|
||||
|
||||
if (xmlBlocks.length === 0) {
|
||||
console.log('------------ USER ------------');
|
||||
console.log('No XML tool invocations found in the input string.');
|
||||
console.log('------------------------------');
|
||||
return;
|
||||
}
|
||||
// Parse input
|
||||
let calls: Array<{ name: string; arguments: Record<string, unknown> }>;
|
||||
|
||||
const executor = XMLToolExecutor.instance();
|
||||
let accumulatedResults = '';
|
||||
|
||||
// Execute tools sequentially and format like ChatBox
|
||||
for (let i = 0; i < xmlBlocks.length; i++) {
|
||||
// Determine tool name from XML block (same as ChatBox lines 148-149)
|
||||
const toolNameMatch = xmlBlocks[i].match(/<([a-zA-Z_][a-zA-Z0-9_-]*)/);
|
||||
const toolName = toolNameMatch ? toolNameMatch[1] : 'unknown_tool';
|
||||
|
||||
try {
|
||||
// Execute single XML block
|
||||
const results = await executor.executeXMLTools(xmlBlocks[i]);
|
||||
const result = results[0]; // Single block should give single result
|
||||
|
||||
if (result) {
|
||||
// Format exactly like ChatBox lines 161-162
|
||||
const formattedResult = `tool: ${toolName}\nsuccess: ${result.success}\nresult:\n${result.result}\n------------\n`;
|
||||
accumulatedResults += formattedResult;
|
||||
}
|
||||
} catch (error) {
|
||||
// Handle individual tool error (same format)
|
||||
const formattedResult = `tool: ${toolName}\nsuccess: false\nresult:\nTool execution failed: ${error}\n------------\n`;
|
||||
accumulatedResults += formattedResult;
|
||||
}
|
||||
}
|
||||
|
||||
// Log accumulated results as USER (what gets sent back to LLM)
|
||||
console.log('------------ USER ------------');
|
||||
console.log(accumulatedResults);
|
||||
console.log('------------------------------');
|
||||
|
||||
// Copy results to clipboard if possible
|
||||
if (navigator.clipboard) {
|
||||
navigator.clipboard.writeText(accumulatedResults).then(() => {
|
||||
console.log("Tool execution results copied to clipboard!");
|
||||
}).catch(() => {
|
||||
console.log("Could not copy to clipboard (requires HTTPS)");
|
||||
});
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.log('------------ USER ------------');
|
||||
console.log(`Error testing XML tool execution: ${error}`);
|
||||
console.log('------------------------------');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Test the AttemptCompletionTool with agent state integration
|
||||
* @param comment - Completion comment to test with
|
||||
*/
|
||||
public async testAttemptCompletion(comment: string): Promise<void> {
|
||||
console.log("🎯 Testing AttemptCompletionTool...");
|
||||
console.log(`📝 Comment: "${comment}"`);
|
||||
|
||||
try {
|
||||
// Get current agent state before test
|
||||
const agentCore = AgentCore.instance();
|
||||
const agentState = agentCore.getAgentState();
|
||||
const initialTaskState = agentState.getIsWorkingOnTask();
|
||||
|
||||
console.log(`📊 Initial agent state:`);
|
||||
console.log(` • isWorkingOnTask: ${initialTaskState}`);
|
||||
|
||||
// Set to working state to test the completion properly
|
||||
if (!initialTaskState) {
|
||||
console.log("🔄 Setting isWorkingOnTask to true for testing...");
|
||||
agentState.setIsWorkingOnTask(true);
|
||||
}
|
||||
|
||||
// Create and execute the tool
|
||||
const completionTool = new AttemptCompletionTool();
|
||||
const result = await completionTool.execute({ comment });
|
||||
|
||||
console.log(`✅ Tool execution result:`);
|
||||
console.log(` • Success: ${result.success}`);
|
||||
console.log(` • Result: ${result.result}`);
|
||||
|
||||
// Check final agent state
|
||||
const finalTaskState = agentState.getIsWorkingOnTask();
|
||||
console.log(`📊 Final agent state:`);
|
||||
console.log(` • isWorkingOnTask: ${finalTaskState}`);
|
||||
|
||||
// Verify state change
|
||||
if (result.success && finalTaskState === false) {
|
||||
console.log("🎉 Success! Agent state correctly updated to not working on task.");
|
||||
} else if (!result.success) {
|
||||
console.log("⚠️ Tool execution failed - state may not have changed.");
|
||||
if (typeof input === 'string') {
|
||||
const parsed = JSON.parse(input);
|
||||
calls = Array.isArray(parsed) ? parsed : [parsed];
|
||||
} else if (Array.isArray(input)) {
|
||||
calls = input as Array<{ name: string; arguments: Record<string, unknown> }>;
|
||||
} else {
|
||||
console.log("⚠️ Warning: State did not change as expected.");
|
||||
calls = [input as { name: string; arguments: Record<string, unknown> }];
|
||||
}
|
||||
|
||||
// Copy result to clipboard if possible
|
||||
if (navigator.clipboard) {
|
||||
const clipboardContent = JSON.stringify(result, null, 2);
|
||||
navigator.clipboard.writeText(clipboardContent).then(() => {
|
||||
console.log("📋 Test results copied to clipboard!");
|
||||
}).catch(() => {
|
||||
console.log("📋 Could not copy to clipboard (requires HTTPS)");
|
||||
});
|
||||
|
||||
console.log(`🔧 Executing ${calls.length} tool call(s)...\n`);
|
||||
|
||||
for (let i = 0; i < calls.length; i++) {
|
||||
const call = calls[i];
|
||||
const toolName = call.name;
|
||||
const toolArgs = call.arguments ?? {};
|
||||
|
||||
console.log(`── Tool call ${i + 1}/${calls.length}: ${toolName}`);
|
||||
console.log(` Arguments: ${JSON.stringify(toolArgs, null, 2)}`);
|
||||
|
||||
const ToolClass = AVAILABLE_TOOLS[toolName as keyof typeof AVAILABLE_TOOLS];
|
||||
if (!ToolClass) {
|
||||
console.error(` ❌ Unknown tool: "${toolName}". Available tools: ${Object.keys(AVAILABLE_TOOLS).join(', ')}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const toolInstance = new ToolClass();
|
||||
const result = await toolInstance.execute(toolArgs);
|
||||
|
||||
// Sync UI state on success
|
||||
if (result.success) {
|
||||
useProjectStore.getState().refreshProjectState();
|
||||
}
|
||||
|
||||
const icon = result.success ? '✅' : '❌';
|
||||
console.log(` ${icon} Success: ${result.success}`);
|
||||
console.log(` Result: ${result.result}\n`);
|
||||
}
|
||||
|
||||
|
||||
console.log('🔧 Tool execution complete.');
|
||||
|
||||
} catch (error) {
|
||||
console.error("❌ Error testing AttemptCompletionTool:", error);
|
||||
console.error('❌ Error in testToolCall:', error);
|
||||
console.log('💡 Expected format: {"name":"tool_name","arguments":{...}}');
|
||||
console.log(' Or an array: [{"name":"tool1","arguments":{...}}, ...]');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -401,8 +342,7 @@ export class KGDebugger {
|
||||
console.log(" debugSelectedItems() - Show info about selected items");
|
||||
console.log(" createTestRegion() - Create test region (not implemented)");
|
||||
console.log(" testExtractXMLFromString(input) - Test XML extraction from string");
|
||||
console.log(" testXMLToolExecution(input) - Test complete XML tool execution pipeline");
|
||||
console.log(" testAttemptCompletion(comment) - Test AttemptCompletionTool with agent state");
|
||||
console.log(" testToolCall(input) - Execute tool call(s) from JSON and show results");
|
||||
console.log(" inputChatBox(content, interval?) - Type into ChatBox textarea and submit with Enter");
|
||||
console.log(" help() - Show this help");
|
||||
console.log("");
|
||||
@@ -410,9 +350,11 @@ export class KGDebugger {
|
||||
console.log(" - Select regions in the DAW first, then run debug methods");
|
||||
console.log(" - Results are logged to console and copied to clipboard when possible");
|
||||
console.log(" - Use browser developer tools for best experience");
|
||||
console.log(" - For XML testing, try: testExtractXMLFromString('I will <add_notes><note>...</note></add_notes> create notes');");
|
||||
console.log(" - For full tool execution, try: await testXMLToolExecution('Create notes: <add_notes><note><pitch>C4</pitch><start_beat>0</start_beat><length>1</length></note></add_notes>');");
|
||||
console.log(" - For completion testing, try: await testAttemptCompletion('Successfully created a C major chord');");
|
||||
console.log("");
|
||||
console.log("💡 testToolCall examples:");
|
||||
console.log(' await KGStudio.KGDebugger.testToolCall(\'{"name":"read_music","arguments":{"start_beat":0,"length":8}}\')');
|
||||
console.log(' await KGStudio.KGDebugger.testToolCall({name:"add_notes",arguments:{notes:[{pitch:"C4",start_beat:0,length:1}]}})');
|
||||
console.log(' await KGStudio.KGDebugger.testToolCall([{name:"remove_notes",arguments:{start_beat:0,end_beat:4}},{name:"read_music",arguments:{}}])');
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user