3dtours/KGStudio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: migrate agent system from XML tool calling to OpenAI SDK with native function calling

Browse Source 
- Replace custom XML-based tool parsing (XMLToolExecutor) with OpenAI SDK's
  native tool_calls via `openai` npm package (dangerouslyAllowBrowser)
- Consolidate 4 LLM providers (OpenAI, Claude, Gemini, ClaudeOpenRouter)
  into a single OpenAI SDK-based LLMProvider compatible with any
  OpenAI-style API (OpenAI, OpenRouter, Ollama, vLLM)
- Move agentic tool execution loop from ChatBox into AgentCore
- Update Message type to support tool roles, tool_calls, and tool_call_id
- Update system prompt to remove XML formatting instructions (~45% smaller)
- Remove AttemptCompletionTool (replaced by stop_reason detection),
  ThinkTool, and ThinkingTool
- Polish tool descriptions for OpenAI function calling schema compliance
- Normalize base URLs by stripping /chat/completions suffix
This commit is contained in:
Xiaohan-Tian
2026-04-05 18:59:25 -07:00
parent 597fb9a292
commit 2cd976ff96
32 changed files with 697 additions and 2310 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/agent/tools/AddNotesTool.ts
+9 -9
View File
@@ -12,35 +12,35 @@ import { KGCore } from '../../core/KGCore';
*/
export class AddNotesTool extends BaseTool {
readonly name = 'add_notes';
readonly description = 'Create one or more MIDI notes in the current region. Each note requires pitch (e.g., "C4", "F#3"), start_beat (beat position), and length (duration in beats).';
readonly description = 'Add one or more MIDI notes to the current region. Use this to create melodies, chords, or any musical content. Notes use absolute beat positions on the project timeline — not relative to the region start.';
readonly parameters: Record<string, ToolParameter> = {
notes: {
type: 'array',
description: 'Array of notes to create',
description: 'List of notes to add. To create a chord, give multiple notes the same start_beat. To create a melody, use sequential start_beat values.',
required: true,
items: {
type: 'object',
description: 'A MIDI note definition',
description: 'A single note',
properties: {
pitch: {
type: 'string',
description: 'Note pitch in scientific notation (e.g., "C4", "F#3", "Bb2")',
description: 'Pitch in scientific notation: note name, optional accidental (# or b), and octave number. Examples: "C4" (middle C), "F#3" (F-sharp 3rd octave), "Bb2" (B-flat 2nd octave).',
required: true
},
start_beat: {
type: 'number',
description: 'Start position in beats (e.g., 0, 1.5, 2)',
description: 'Absolute beat position on the project timeline where the note starts. This is NOT relative to the region — beat 6 means beat 6 in the project regardless of where the region begins. Fractional values are supported (e.g., 0.5 = half a beat after beat 0).',
required: true
},
length: {
type: 'number',
description: 'Note duration in beats (e.g., 1, 0.5, 4)',
description: 'Duration of the note in beats. In 4/4 time: 4 = whole note, 2 = half note, 1 = quarter note, 0.5 = eighth note, 0.25 = sixteenth note.',
required: true
},
velocity: {
type: 'number',
description: 'Note velocity (1-127, default: 127)',
description: 'Note velocity / loudness from 1 (softest) to 127 (loudest). Defaults to 127 if omitted.',
required: false
}
}
@@ -48,7 +48,7 @@ export class AddNotesTool extends BaseTool {
},
region_id: {
type: 'string',
description: 'ID of the region to add notes to. If not provided, uses the currently selected region.',
description: 'Target region ID. If omitted, uses the currently active piano roll region or selected region.',
required: false
}
};
src/agent/tools/AttemptCompletionTool.ts
-49
View File
@@ -1,49 +0,0 @@
import { BaseTool } from './BaseTool';
import type { ToolResult, ToolParameter } from './BaseTool';
import { AgentCore } from '../core/AgentCore';
/**
* Tool for signaling task completion
* This is a pure agent state tool that doesn't modify the DAW but signals
* to the agent system that the user's requested task has been completed
*/
export class AttemptCompletionTool extends BaseTool {
readonly name = 'attempt_completion';
readonly description = 'Signal that the current user task is fully complete. Only use this when you have successfully fulfilled all aspects of the user\'s request.';
readonly parameters: Record<string, ToolParameter> = {
comment: {
type: 'string',
description: 'A brief comment describing what was completed and any relevant details about the task fulfillment.',
required: true
}
};
async execute(params: Record<string, unknown>): Promise<ToolResult> {
try {
// Validate parameters
this.validateParameters(params);
const comment = params.comment as string;
// Validate comment is not empty
if (!comment.trim()) {
return this.createErrorResult('Comment cannot be empty. Please provide a meaningful completion summary.');
}
// Get current agent state and update task completion status
const agentCore = AgentCore.instance();
const agentState = agentCore.getAgentState();
// Mark that we're no longer working on a task
agentState.setIsWorkingOnTask(false);
return this.createSuccessResult(
`Task completed: ${comment}. `
);
} catch (error) {
return this.createErrorResult(`Failed to mark task as complete: ${error}`);
}
}
}
src/agent/tools/BaseTool.ts
+83 -5
View File
@@ -21,7 +21,28 @@ export interface ToolParameter {
}
/**
* Tool definition schema
* OpenAI-compatible JSON Schema for function parameters
*/
export interface OpenAIFunctionParameters {
type: 'object';
properties: Record<string, unknown>;
required?: string[];
}
/**
* OpenAI-compatible tool definition
*/
export interface OpenAIToolDefinition {
type: 'function';
function: {
name: string;
description: string;
parameters: OpenAIFunctionParameters;
};
}
/**
* Tool definition schema (internal format)
*/
export interface ToolDefinition {
name: string;
@@ -48,14 +69,71 @@ export abstract class BaseTool {
/**
* Get the tool definition in OpenAI function calling format
*/
getDefinition(): ToolDefinition {
getDefinition(): OpenAIToolDefinition {
return {
name: this.name,
description: this.description,
parameters: this.parameters
type: 'function',
function: {
name: this.name,
description: this.description,
parameters: this.convertToJsonSchema(this.parameters)
}
};
}
/**
* Convert internal ToolParameter map to OpenAI-compatible JSON Schema
*/
private convertToJsonSchema(params: Record<string, ToolParameter>): OpenAIFunctionParameters {
const properties: Record<string, unknown> = {};
const required: string[] = [];
for (const [name, param] of Object.entries(params)) {
properties[name] = this.convertParamToJsonSchema(param);
if (param.required) {
required.push(name);
}
}
return {
type: 'object',
properties,
...(required.length > 0 ? { required } : {})
};
}
/**
* Convert a single ToolParameter to JSON Schema format
*/
private convertParamToJsonSchema(param: ToolParameter): Record<string, unknown> {
const schema: Record<string, unknown> = {
type: param.type,
description: param.description
};
if (param.type === 'array' && param.items) {
schema.items = this.convertParamToJsonSchema(param.items);
}
if (param.type === 'object' && param.properties) {
const properties: Record<string, unknown> = {};
const required: string[] = [];
for (const [name, prop] of Object.entries(param.properties)) {
properties[name] = this.convertParamToJsonSchema(prop);
if (prop.required) {
required.push(name);
}
}
schema.properties = properties;
if (required.length > 0) {
schema.required = required;
}
}
return schema;
}
/**
* Validate parameters against the tool's parameter schema
* @param params Parameters to validate
src/agent/tools/ReadMusicTool.ts
+5 -5
View File
@@ -12,22 +12,22 @@ import { FLUIDR3_INSTRUMENT_MAP } from '../../constants/generalMidiConstants';
*/
export class ReadMusicTool extends BaseTool {
readonly name = 'read_music';
readonly description = 'Read the music content from a specific track or all tracks, returning the content in ABC notation format.';
readonly description = 'Read existing musical content from one or more tracks, returned as ABC notation. Use this to understand what notes already exist before making edits. Always call this before asking the user about their music. The output is bar-aligned and includes key/time signature headers.';
readonly parameters: Record<string, ToolParameter> = {
track_id: {
type: 'string',
description: 'The track ID to read, or "all" to read all tracks. If not provided, reads the first available track.',
description: 'Which track to read. Pass a specific track ID, or "all" to read every track. If omitted, reads the first available track.',
required: false
},
start_beat: {
type: 'number',
description: 'Start beat position to read from (default: 0)',
description: 'Absolute beat position to start reading from. The actual output will be rounded down to the nearest bar boundary. Defaults to 0.',
required: false
},
length: {
type: 'number',
description: 'Length in beats to read (default: entire track/project)',
description: 'Number of beats to read. The actual output will be rounded up to the nearest bar boundary. If omitted, reads to the end of the track.',
required: false
}
};
src/agent/tools/RemoveNotesTool.ts
+5 -5
View File
@@ -11,22 +11,22 @@ import { KGCore } from '../../core/KGCore';
*/
export class RemoveNotesTool extends BaseTool {
readonly name = 'remove_notes';
readonly description = 'Remove MIDI notes from the current region within a specified beat range. All notes that start within the range will be deleted.';
readonly description = 'Remove all MIDI notes whose start position falls within the specified beat range. Use this to clear a section before rewriting it, or to delete unwanted notes. Beat positions are absolute on the project timeline.';
readonly parameters: Record<string, ToolParameter> = {
start_beat: {
type: 'number',
description: 'Start of the beat range to remove notes from (inclusive)',
description: 'Absolute beat position where the removal range begins (inclusive). A note starting at exactly this beat will be removed.',
required: true
},
end_beat: {
type: 'number',
description: 'End of the beat range to remove notes from (exclusive)',
description: 'Absolute beat position where the removal range ends (exclusive). A note starting at exactly this beat will NOT be removed. Must be greater than start_beat.',
required: true
},
region_id: {
type: 'string',
description: 'ID of the region to remove notes from. If not provided, uses the currently selected region.',
description: 'Target region ID. If omitted, uses the currently active piano roll region or selected region.',
required: false
}
};
src/agent/tools/ThinkTool.ts
-35
View File
@@ -1,35 +0,0 @@
import { BaseTool } from './BaseTool';
import type { ToolResult, ToolParameter } from './BaseTool';
/**
* Pseudo tool for handling <think> tags in LLM responses
* This tool displays the thinking content in the UI but doesn't send results back to the LLM
* Handles XML format: <think>any content here</think>
* This is functionally identical to ThinkingTool but handles the shorter tag name
*/
export class ThinkTool extends BaseTool {
readonly name = 'think';
readonly description = 'Pseudo tool for handling LLM thinking content from <think> tags. Shows content in UI but does not send results back to LLM.';
readonly parameters: Record<string, ToolParameter> = {
content: {
type: 'string',
description: 'The thinking content from the XML tag',
required: false
}
};
async execute(params: Record<string, unknown>): Promise<ToolResult> {
try {
// Extract the thinking content from the parameters
// const content = params.content as string || '';
// Return the thinking content as a successful result
// This will be displayed in the UI but not sent back to the LLM
return this.createSuccessResult("Thinking completed.");
} catch (error) {
return this.createErrorResult(`Failed to process thinking content: ${error}`);
}
}
}
src/agent/tools/ThinkingTool.ts
-34
View File
@@ -1,34 +0,0 @@
import { BaseTool } from './BaseTool';
import type { ToolResult, ToolParameter } from './BaseTool';
/**
* Pseudo tool for handling <thinking> tags in LLM responses
* This tool displays the thinking content in the UI but doesn't send results back to the LLM
* Handles XML format: <thinking>any content here</thinking>
*/
export class ThinkingTool extends BaseTool {
readonly name = 'thinking';
readonly description = 'Pseudo tool for handling LLM thinking content. Shows content in UI but does not send results back to LLM.';
readonly parameters: Record<string, ToolParameter> = {
content: {
type: 'string',
description: 'The thinking content from the XML tag',
required: false
}
};
async execute(params: Record<string, unknown>): Promise<ToolResult> {
try {
// Extract the thinking content from the parameters
// const content = params.content as string || '';
// Return the thinking content as a successful result
// This will be displayed in the UI but not sent back to the LLM
return this.createSuccessResult("Thinking completed.");
} catch (error) {
return this.createErrorResult(`Failed to process thinking content: ${error}`);
}
}
}
src/agent/tools/index.ts
+3 -9
View File
@@ -1,25 +1,19 @@
// Base tool system
export { BaseTool } from './BaseTool';
export type { ToolResult, ToolParameter, ToolDefinition } from './BaseTool';
export type { ToolResult, ToolParameter, ToolDefinition, OpenAIToolDefinition, OpenAIFunctionParameters } from './BaseTool';
// Specific tools
import { AddNotesTool } from './AddNotesTool';
import { RemoveNotesTool } from './RemoveNotesTool';
import { ReadMusicTool } from './ReadMusicTool';
import { AttemptCompletionTool } from './AttemptCompletionTool';
import { ThinkingTool } from './ThinkingTool';
import { ThinkTool } from './ThinkTool';
export { AddNotesTool, RemoveNotesTool, ReadMusicTool, AttemptCompletionTool, ThinkingTool, ThinkTool };
export { AddNotesTool, RemoveNotesTool, ReadMusicTool };
// Tool registry for easy access
export const AVAILABLE_TOOLS = {
add_notes: AddNotesTool,
remove_notes: RemoveNotesTool,
read_music: ReadMusicTool,
attempt_completion: AttemptCompletionTool,
thinking: ThinkingTool,
think: ThinkTool
} as const;
export type ToolName = keyof typeof AVAILABLE_TOOLS;
export type ToolName = keyof typeof AVAILABLE_TOOLS;