refactor: migrate agent system from XML tool calling to OpenAI SDK with native function calling

- Replace custom XML-based tool parsing (XMLToolExecutor) with OpenAI SDK's
  native tool_calls via `openai` npm package (dangerouslyAllowBrowser)
- Consolidate 4 LLM providers (OpenAI, Claude, Gemini, ClaudeOpenRouter)
  into a single OpenAI SDK-based LLMProvider compatible with any
  OpenAI-style API (OpenAI, OpenRouter, Ollama, vLLM)
- Move agentic tool execution loop from ChatBox into AgentCore
- Update Message type to support tool roles, tool_calls, and tool_call_id
- Update system prompt to remove XML formatting instructions (~45% smaller)
- Remove AttemptCompletionTool (replaced by stop_reason detection),
  ThinkTool, and ThinkingTool
- Polish tool descriptions for OpenAI function calling schema compliance
- Normalize base URLs by stripping /chat/completions suffix
This commit is contained in:
Xiaohan-Tian
2026-04-05 18:59:25 -07:00
parent 597fb9a292
commit 2cd976ff96
32 changed files with 697 additions and 2310 deletions
+67 -125
View File
@@ -7,9 +7,8 @@ import { KGCore } from './KGCore';
import { KGMidiRegion } from './region/KGMidiRegion';
import { convertRegionToABCNotation } from '../util/abcNotationUtil';
import { extractXMLFromString } from '../util/xmlUtil';
import { XMLToolExecutor } from '../agent/core/XMLToolExecutor';
import { AgentCore } from '../agent/core/AgentCore';
import { AttemptCompletionTool } from '../agent/tools/AttemptCompletionTool';
import { AVAILABLE_TOOLS } from '../agent/tools';
import type { TimeSignature } from '../types/projectTypes';
import { useProjectStore } from '../stores/projectStore';
@@ -28,8 +27,7 @@ export class KGDebugger {
'debugSelectedItems()',
'createTestRegion()',
'testExtractXMLFromString(input)',
'testXMLToolExecution(input)',
'testAttemptCompletion(comment)',
'testToolCall(jsonInput)',
'inputChatBox(content, interval?)'
]);
}
@@ -264,129 +262,72 @@ export class KGDebugger {
}
/**
* Test the complete XML tool execution pipeline
* @param input - String containing XML tool invocations to execute
* Test native tool calling by executing tool calls from a JSON string.
* Accepts a single tool call object or an array of tool call objects.
*
* Usage examples in browser console:
*
* // Single tool call:
* await KGStudio.KGDebugger.testToolCall('{"name":"read_music","arguments":{"start_beat":0,"length":8}}')
*
* // Multiple tool calls:
* await KGStudio.KGDebugger.testToolCall('[{"name":"remove_notes","arguments":{"start_beat":0,"end_beat":4}},{"name":"add_notes","arguments":{"notes":[{"pitch":"C4","start_beat":0,"length":1}]}}]')
*
* // Can also pass a JS object directly (no need to stringify):
* await KGStudio.KGDebugger.testToolCall({name:"read_music",arguments:{start_beat:0}})
*
* @param input - JSON string, object, or array of tool call(s).
* Each tool call should have: { name: string, arguments: object }
*/
public async testXMLToolExecution(input: string): Promise<void> {
console.log('------------ ASSISTANT ------------');
console.log(input);
console.log('-----------------------------------');
public async testToolCall(input: string | Record<string, unknown> | Record<string, unknown>[]): Promise<void> {
try {
// Extract XML blocks first to get tool names (same logic as ChatBox)
const xmlBlocks = extractXMLFromString(input);
if (xmlBlocks.length === 0) {
console.log('------------ USER ------------');
console.log('No XML tool invocations found in the input string.');
console.log('------------------------------');
return;
}
// Parse input
let calls: Array<{ name: string; arguments: Record<string, unknown> }>;
const executor = XMLToolExecutor.instance();
let accumulatedResults = '';
// Execute tools sequentially and format like ChatBox
for (let i = 0; i < xmlBlocks.length; i++) {
// Determine tool name from XML block (same as ChatBox lines 148-149)
const toolNameMatch = xmlBlocks[i].match(/<([a-zA-Z_][a-zA-Z0-9_-]*)/);
const toolName = toolNameMatch ? toolNameMatch[1] : 'unknown_tool';
try {
// Execute single XML block
const results = await executor.executeXMLTools(xmlBlocks[i]);
const result = results[0]; // Single block should give single result
if (result) {
// Format exactly like ChatBox lines 161-162
const formattedResult = `tool: ${toolName}\nsuccess: ${result.success}\nresult:\n${result.result}\n------------\n`;
accumulatedResults += formattedResult;
}
} catch (error) {
// Handle individual tool error (same format)
const formattedResult = `tool: ${toolName}\nsuccess: false\nresult:\nTool execution failed: ${error}\n------------\n`;
accumulatedResults += formattedResult;
}
}
// Log accumulated results as USER (what gets sent back to LLM)
console.log('------------ USER ------------');
console.log(accumulatedResults);
console.log('------------------------------');
// Copy results to clipboard if possible
if (navigator.clipboard) {
navigator.clipboard.writeText(accumulatedResults).then(() => {
console.log("Tool execution results copied to clipboard!");
}).catch(() => {
console.log("Could not copy to clipboard (requires HTTPS)");
});
}
} catch (error) {
console.log('------------ USER ------------');
console.log(`Error testing XML tool execution: ${error}`);
console.log('------------------------------');
}
}
/**
* Test the AttemptCompletionTool with agent state integration
* @param comment - Completion comment to test with
*/
public async testAttemptCompletion(comment: string): Promise<void> {
console.log("🎯 Testing AttemptCompletionTool...");
console.log(`📝 Comment: "${comment}"`);
try {
// Get current agent state before test
const agentCore = AgentCore.instance();
const agentState = agentCore.getAgentState();
const initialTaskState = agentState.getIsWorkingOnTask();
console.log(`📊 Initial agent state:`);
console.log(` • isWorkingOnTask: ${initialTaskState}`);
// Set to working state to test the completion properly
if (!initialTaskState) {
console.log("🔄 Setting isWorkingOnTask to true for testing...");
agentState.setIsWorkingOnTask(true);
}
// Create and execute the tool
const completionTool = new AttemptCompletionTool();
const result = await completionTool.execute({ comment });
console.log(`✅ Tool execution result:`);
console.log(` • Success: ${result.success}`);
console.log(` • Result: ${result.result}`);
// Check final agent state
const finalTaskState = agentState.getIsWorkingOnTask();
console.log(`📊 Final agent state:`);
console.log(` • isWorkingOnTask: ${finalTaskState}`);
// Verify state change
if (result.success && finalTaskState === false) {
console.log("🎉 Success! Agent state correctly updated to not working on task.");
} else if (!result.success) {
console.log("⚠️ Tool execution failed - state may not have changed.");
if (typeof input === 'string') {
const parsed = JSON.parse(input);
calls = Array.isArray(parsed) ? parsed : [parsed];
} else if (Array.isArray(input)) {
calls = input as Array<{ name: string; arguments: Record<string, unknown> }>;
} else {
console.log("⚠️ Warning: State did not change as expected.");
calls = [input as { name: string; arguments: Record<string, unknown> }];
}
// Copy result to clipboard if possible
if (navigator.clipboard) {
const clipboardContent = JSON.stringify(result, null, 2);
navigator.clipboard.writeText(clipboardContent).then(() => {
console.log("📋 Test results copied to clipboard!");
}).catch(() => {
console.log("📋 Could not copy to clipboard (requires HTTPS)");
});
console.log(`🔧 Executing ${calls.length} tool call(s)...\n`);
for (let i = 0; i < calls.length; i++) {
const call = calls[i];
const toolName = call.name;
const toolArgs = call.arguments ?? {};
console.log(`── Tool call ${i + 1}/${calls.length}: ${toolName}`);
console.log(` Arguments: ${JSON.stringify(toolArgs, null, 2)}`);
const ToolClass = AVAILABLE_TOOLS[toolName as keyof typeof AVAILABLE_TOOLS];
if (!ToolClass) {
console.error(` ❌ Unknown tool: "${toolName}". Available tools: ${Object.keys(AVAILABLE_TOOLS).join(', ')}`);
continue;
}
const toolInstance = new ToolClass();
const result = await toolInstance.execute(toolArgs);
// Sync UI state on success
if (result.success) {
useProjectStore.getState().refreshProjectState();
}
const icon = result.success ? '✅' : '❌';
console.log(` ${icon} Success: ${result.success}`);
console.log(` Result: ${result.result}\n`);
}
console.log('🔧 Tool execution complete.');
} catch (error) {
console.error("❌ Error testing AttemptCompletionTool:", error);
console.error('❌ Error in testToolCall:', error);
console.log('💡 Expected format: {"name":"tool_name","arguments":{...}}');
console.log(' Or an array: [{"name":"tool1","arguments":{...}}, ...]');
}
}
@@ -401,8 +342,7 @@ export class KGDebugger {
console.log(" debugSelectedItems() - Show info about selected items");
console.log(" createTestRegion() - Create test region (not implemented)");
console.log(" testExtractXMLFromString(input) - Test XML extraction from string");
console.log(" testXMLToolExecution(input) - Test complete XML tool execution pipeline");
console.log(" testAttemptCompletion(comment) - Test AttemptCompletionTool with agent state");
console.log(" testToolCall(input) - Execute tool call(s) from JSON and show results");
console.log(" inputChatBox(content, interval?) - Type into ChatBox textarea and submit with Enter");
console.log(" help() - Show this help");
console.log("");
@@ -410,9 +350,11 @@ export class KGDebugger {
console.log(" - Select regions in the DAW first, then run debug methods");
console.log(" - Results are logged to console and copied to clipboard when possible");
console.log(" - Use browser developer tools for best experience");
console.log(" - For XML testing, try: testExtractXMLFromString('I will <add_notes><note>...</note></add_notes> create notes');");
console.log(" - For full tool execution, try: await testXMLToolExecution('Create notes: <add_notes><note><pitch>C4</pitch><start_beat>0</start_beat><length>1</length></note></add_notes>');");
console.log(" - For completion testing, try: await testAttemptCompletion('Successfully created a C major chord');");
console.log("");
console.log("💡 testToolCall examples:");
console.log(' await KGStudio.KGDebugger.testToolCall(\'{"name":"read_music","arguments":{"start_beat":0,"length":8}}\')');
console.log(' await KGStudio.KGDebugger.testToolCall({name:"add_notes",arguments:{notes:[{pitch:"C4",start_beat:0,length:1}]}})');
console.log(' await KGStudio.KGDebugger.testToolCall([{name:"remove_notes",arguments:{start_beat:0,end_beat:4}},{name:"read_music",arguments:{}}])');
}
/**
+1 -1
View File
@@ -190,7 +190,7 @@ export class ConfigManager {
},
claude_openrouter: {
api_key: '',
base_url: 'https://openrouter.ai/api/v1/chat/completions',
base_url: 'https://openrouter.ai/api/v1',
model: 'anthropic/claude-sonnet-4.5'
},
openai_compatible: {