From cf4f3edb30c91dfa80468901237c08544a0f0365 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Sat, 16 Aug 2025 15:49:41 -0700 Subject: [PATCH 1/7] removed native tool_calls handling from `OpenAIProvider`. --- src/agent/llm/OpenAIProvider.ts | 184 ++------------------------------ 1 file changed, 11 insertions(+), 173 deletions(-) diff --git a/src/agent/llm/OpenAIProvider.ts b/src/agent/llm/OpenAIProvider.ts index 92f19e0..3e10f0a 100644 --- a/src/agent/llm/OpenAIProvider.ts +++ b/src/agent/llm/OpenAIProvider.ts @@ -66,30 +66,15 @@ export class OpenAIProvider extends LLMProvider { /** * Parse Ollama's raw JSON chunk format */ - private parseOllamaChunk(chunk: string): { thinking?: string; content?: string; isDone?: boolean; toolCalls?: Array<{ name: string; arguments: Record }> } { + private parseOllamaChunk(chunk: string): { thinking?: string; content?: string; isDone?: boolean } { try { const json = JSON.parse(chunk.trim()); const thinking: string | undefined = json.message?.thinking; const content: string | undefined = json.message?.content || json.response; // Handle both chat and completion formats - type WireToolCall = { function?: { name?: unknown; arguments?: unknown } }; - const toolCallsRaw: unknown[] | undefined = json.message?.tool_calls as unknown[] | undefined; - const toolCalls = Array.isArray(toolCallsRaw) - ? (toolCallsRaw - .map((tc: unknown) => { - const wire = tc as WireToolCall; - const fn = wire?.function; - if (!fn || typeof fn.name !== 'string') return null; - const args = fn.arguments; - if (args === null || typeof args !== 'object' || Array.isArray(args)) return null; - return { name: fn.name, arguments: args as Record }; - }) - .filter((v): v is { name: string; arguments: Record } => v !== null)) - : undefined; return { thinking, content, - isDone: json.done === true, - toolCalls + isDone: json.done === true }; } catch { return {}; // Invalid JSON, return empty object @@ -99,7 +84,7 @@ export class OpenAIProvider extends LLMProvider { /** * Parse OpenAI's SSE format chunk */ - private parseOpenAIChunk(line: string): { thinking?: string; content?: string; isDone?: boolean; toolCallDelta?: Array<{ index?: number; function?: { name?: string; arguments?: string } }> } { + private parseOpenAIChunk(line: string): { thinking?: string; content?: string; isDone?: boolean } { if (!line.startsWith('data: ')) { return {}; } @@ -114,11 +99,7 @@ export class OpenAIProvider extends LLMProvider { const delta = json.choices?.[0]?.delta; const thinking: string | undefined = delta?.thinking; // Some providers may stream "thinking" const content: string | undefined = delta?.content; - const tcd = delta?.tool_calls; - const toolCallDelta: Array<{ index?: number; function?: { name?: string; arguments?: string } }> | undefined = Array.isArray(tcd) - ? (tcd as Array<{ index?: number; function?: { name?: string; arguments?: string } }>) - : undefined; - return { thinking, content, isDone: false, toolCallDelta }; + return { thinking, content, isDone: false }; } catch { return {}; // Skip invalid JSON lines } @@ -126,8 +107,7 @@ export class OpenAIProvider extends LLMProvider { async *generateStream( messages: Message[], - systemPrompt?: string, - tools?: Record[] + systemPrompt?: string ): AsyncIterableIterator { // Get fresh config values const config = this.getCurrentConfig(); @@ -157,7 +137,6 @@ export class OpenAIProvider extends LLMProvider { ...(config.flexMode && !config.isCompatibleProvider ? { service_tier: 'flex' } : {}), messages: openAIMessages, stream: true, - tools: tools || undefined }), }); @@ -174,28 +153,7 @@ export class OpenAIProvider extends LLMProvider { let buffer = ''; let firstChunkProcessed = false; let lastSegmentType: 'thinking' | 'content' | null = null; - const pendingFunctionCalls: Array<{ name: string; arguments: Record }> = []; - const openAIToolCallBuilders: Record = {}; - const escapeXml = (text: string): string => - String(text) - .replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"') - .replace(/'/g, '''); - - const functionCallToXml = (name: string, args: Record): string => { - const keys = Object.keys(args); - const inner = keys - .map((k) => { - const value = (args as Record)[k]; - const text = typeof value === 'string' ? value : JSON.stringify(value); - return `<${k}>${escapeXml(text)}`; - }) - .join('\n'); - return `<${name}>\n${inner}\n`; - }; try { while (true) { @@ -220,19 +178,8 @@ export class OpenAIProvider extends LLMProvider { } if (this.isOllamaFormat) { - const { thinking, content, isDone, toolCalls } = this.parseOllamaChunk(trimmedLine); + const { thinking, content, isDone } = this.parseOllamaChunk(trimmedLine); if (isDone) { - // Append any pending or current tool calls as XML before finishing - const allToolCalls = [ - ...pendingFunctionCalls, - ...(toolCalls || []) - ]; - if (allToolCalls.length > 0) { - const xmlBlocks = allToolCalls - .map((tc) => `\n${functionCallToXml(tc.name, tc.arguments)}\n`) - .join(''); - yield { type: 'text', content: xmlBlocks }; - } yield { type: 'done', content: '' }; return; } @@ -253,41 +200,11 @@ export class OpenAIProvider extends LLMProvider { lastSegmentType = 'content'; } - if (Array.isArray(toolCalls) && toolCalls.length > 0) { - // Accumulate and append at the end of stream - pendingFunctionCalls.push(...toolCalls); - } continue; } - const { thinking, content, isDone, toolCallDelta } = this.parseOpenAIChunk(trimmedLine); + const { thinking, content, isDone } = this.parseOpenAIChunk(trimmedLine); if (isDone) { - // Finalize any accumulated OpenAI tool calls and emit XML - const finalizedToolCalls: Array<{ name: string; arguments: Record }> = []; - for (const indexStr of Object.keys(openAIToolCallBuilders)) { - const idx = Number(indexStr); - const builder = openAIToolCallBuilders[idx]; - if (!builder || !builder.name) continue; - let argsObj: Record | null = null; - if (builder.argumentsText) { - try { - argsObj = JSON.parse(builder.argumentsText); - } catch { - argsObj = null; - } - } - if (argsObj) { - finalizedToolCalls.push({ name: builder.name, arguments: argsObj }); - } - } - - const allToolCalls = [...pendingFunctionCalls, ...finalizedToolCalls]; - if (allToolCalls.length > 0) { - const xmlBlocks = allToolCalls - .map((tc) => `\n${functionCallToXml(tc.name, tc.arguments)}\n`) - .join(''); - yield { type: 'text', content: xmlBlocks }; - } yield { type: 'done', content: '' }; return; } @@ -307,24 +224,6 @@ export class OpenAIProvider extends LLMProvider { yield { type: 'text', content }; lastSegmentType = 'content'; } - - if (Array.isArray(toolCallDelta) && toolCallDelta.length > 0) { - for (const tc of toolCallDelta) { - const index: number = typeof tc.index === 'number' ? tc.index : 0; - if (!openAIToolCallBuilders[index]) { - openAIToolCallBuilders[index] = { argumentsText: '' }; - } - const fn = tc.function; - if (fn) { - if (typeof fn.name === 'string') { - openAIToolCallBuilders[index].name = fn.name; - } - if (typeof fn.arguments === 'string') { - openAIToolCallBuilders[index].argumentsText += fn.arguments; - } - } - } - } } } } finally { @@ -334,8 +233,7 @@ export class OpenAIProvider extends LLMProvider { async generateCompletion( messages: Message[], - systemPrompt?: string, - tools?: Record[] + systemPrompt?: string ): Promise { // Get fresh config values const config = this.getCurrentConfig(); @@ -365,7 +263,6 @@ export class OpenAIProvider extends LLMProvider { ...(config.flexMode && !config.isCompatibleProvider ? { service_tier: 'flex' } : {}), messages: openAIMessages, stream: false, - tools: tools || undefined }), }); @@ -376,25 +273,6 @@ export class OpenAIProvider extends LLMProvider { const data = await response.json(); // Handle different response formats - const escapeXml = (text: string): string => - String(text) - .replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"') - .replace(/'/g, '''); - - const functionCallToXml = (name: string, args: Record): string => { - const keys = Object.keys(args); - const inner = keys - .map((k) => { - const value = (args as Record)[k]; - const text = typeof value === 'string' ? value : JSON.stringify(value); - return `<${k}>${escapeXml(text)}`; - }) - .join('\n'); - return `<${name}>\n${inner}\n`; - }; if (data.message) { // Ollama/compatible format @@ -402,28 +280,8 @@ export class OpenAIProvider extends LLMProvider { const contentText: string = data.message.content || data.response || ''; const textCombined = thinking && contentText ? `${thinking}\n\n${contentText}` : (thinking || contentText); - type WireToolCall = { function?: { name?: unknown; arguments?: unknown } }; - const toolCallsRaw: unknown[] | undefined = data.message.tool_calls as unknown[] | undefined; - const toolCalls: Array<{ name: string; arguments: Record }> = Array.isArray(toolCallsRaw) - ? (toolCallsRaw - .map((tc: unknown) => { - const wire = tc as WireToolCall; - const fn = wire?.function; - if (!fn || typeof fn.name !== 'string') return null; - const args = fn.arguments; - if (args === null || typeof args !== 'object' || Array.isArray(args)) return null; - return { name: fn.name, arguments: args as Record }; - }) - .filter((v): v is { name: string; arguments: Record } => v !== null)) - : []; - - const xmlBlocks = toolCalls.length > 0 - ? toolCalls.map((tc) => `\n${functionCallToXml(tc.name, tc.arguments)}\n`).join('') - : ''; - return { - content: `${textCombined}${xmlBlocks}`, - toolCalls: data.message.tool_calls || undefined, + content: textCombined, finished: data.done === true || data.done_reason === 'stop' }; } else { @@ -433,29 +291,9 @@ export class OpenAIProvider extends LLMProvider { const contentText: string = choice?.message?.content || ''; const textCombined = thinking && contentText ? `${thinking}\n\n${contentText}` : (thinking || contentText); - type WireToolCall2 = { function?: { name?: unknown; arguments?: unknown } }; - const toolCallsRaw: unknown[] | undefined = choice?.message?.tool_calls as unknown[] | undefined; - const toolCalls: Array<{ name: string; arguments: Record }> = Array.isArray(toolCallsRaw) - ? (toolCallsRaw - .map((tc: unknown) => { - const wire = tc as WireToolCall2; - const fn = wire?.function; - if (!fn || typeof fn.name !== 'string') return null; - const args = fn.arguments; - if (args === null || typeof args !== 'object' || Array.isArray(args)) return null; - return { name: fn.name, arguments: args as Record }; - }) - .filter((v): v is { name: string; arguments: Record } => v !== null)) - : []; - - const xmlBlocks = toolCalls.length > 0 - ? toolCalls.map((tc) => `\n${functionCallToXml(tc.name, tc.arguments)}\n`).join('') - : ''; - return { - content: `${textCombined}${xmlBlocks}`, - toolCalls: choice?.message?.tool_calls || undefined, - finished: choice?.finish_reason === 'stop' || choice?.finish_reason === 'tool_calls' + content: textCombined, + finished: choice?.finish_reason === 'stop' }; } } From 429efaf629e7c538c611c63ba45e8d21c0ad2a12 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Sat, 16 Aug 2025 16:25:45 -0700 Subject: [PATCH 2/7] cleaned up the LLM providers. --- src/agent/core/AgentCore.ts | 25 ---- src/agent/llm/ClaudeProvider.ts | 69 +--------- src/agent/llm/GeminiProvider.ts | 64 +-------- src/agent/llm/LLMProvider.ts | 14 +- src/agent/llm/OpenAIProvider.ts | 237 ++++++++++++-------------------- src/constants/llmConstants.ts | 10 ++ 6 files changed, 100 insertions(+), 319 deletions(-) create mode 100644 src/constants/llmConstants.ts diff --git a/src/agent/core/AgentCore.ts b/src/agent/core/AgentCore.ts index 6c4a213..66ea05a 100644 --- a/src/agent/core/AgentCore.ts +++ b/src/agent/core/AgentCore.ts @@ -93,31 +93,6 @@ export class AgentCore { } } - /** - * Process user input and get complete response (non-streaming) - */ - async processUserInputComplete(userInput: string): Promise { - if (!this.llmProvider) { - throw new Error('No LLM provider configured'); - } - - // Add user message to state - this.agentState.addMessage('user', userInput); - - // Get system prompt with current context - const systemPrompt = await SystemPrompts.getSystemPromptWithContext(); - - // Get full conversation history with preserved roles - const conversationHistory = this.agentState.getMessages(); - - // Generate complete response with full conversation context - const response = await this.llmProvider.generateCompletion(conversationHistory, systemPrompt); - - // Add assistant response to state - this.agentState.addMessage('assistant', response.content); - - return response.content; - } /** * Abort the current streaming request and clean up messages diff --git a/src/agent/llm/ClaudeProvider.ts b/src/agent/llm/ClaudeProvider.ts index 9b5faa6..e9ec58a 100644 --- a/src/agent/llm/ClaudeProvider.ts +++ b/src/agent/llm/ClaudeProvider.ts @@ -1,5 +1,5 @@ import { LLMProvider } from './LLMProvider'; -import type { StreamChunk, LLMResponse } from './StreamingTypes'; +import type { StreamChunk } from './StreamingTypes'; import type { Message } from '../core/AgentState'; import { ConfigManager } from '../../core/config/ConfigManager'; @@ -159,71 +159,4 @@ export class ClaudeProvider extends LLMProvider { } } - async generateCompletion( - messages: Message[], - systemPrompt?: string, - tools?: Record[] - ): Promise { - const { system, messages: claudeMessages } = this.convertMessages(messages, systemPrompt); - - const requestBody: { - model: string; - max_tokens: number; - messages: Array<{ role: 'user' | 'assistant'; content: string }>; - stream: boolean; - system?: string; - tools?: Record[]; - } = { - model: this.model, - max_tokens: 8192, - messages: claudeMessages, - stream: false - }; - - if (system) { - requestBody.system = system; - } - - if (tools && tools.length > 0) { - requestBody.tools = tools; - } - - const response = await fetch(this.apiEndpoint, { - method: 'POST', - headers: { - 'x-api-key': this.apiKey, - 'Content-Type': 'application/json', - 'anthropic-version': '2023-06-01' - }, - body: JSON.stringify(requestBody), - mode: 'cors' - }); - - if (!response.ok) { - throw new Error(`Claude API error: ${response.status} ${response.statusText}`); - } - - const data = await response.json(); - - // Extract content from Claude's response format - const content = data.content - ?.filter((block: { type: string }) => block.type === 'text') - ?.map((block: { text: string }) => block.text) - ?.join('') || ''; - - // Extract tool calls if present - const toolCalls = data.content - ?.filter((block: { type: string }) => block.type === 'tool_use') - ?.map((block: { id: string; name: string; input: Record }) => ({ - id: block.id, - name: block.name, - parameters: block.input - })) || []; - - return { - content, - toolCalls: toolCalls.length > 0 ? toolCalls : undefined, - finished: data.stop_reason === 'end_turn' || data.stop_reason === 'tool_use' - }; - } } \ No newline at end of file diff --git a/src/agent/llm/GeminiProvider.ts b/src/agent/llm/GeminiProvider.ts index 7a770b1..c62f0c5 100644 --- a/src/agent/llm/GeminiProvider.ts +++ b/src/agent/llm/GeminiProvider.ts @@ -1,5 +1,5 @@ import { LLMProvider } from './LLMProvider'; -import type { StreamChunk, LLMResponse } from './StreamingTypes'; +import type { StreamChunk } from './StreamingTypes'; import type { Message } from '../core/AgentState'; import { ConfigManager } from '../../core/config/ConfigManager'; @@ -180,66 +180,4 @@ export class GeminiProvider extends LLMProvider { reader.releaseLock(); } } - - async generateCompletion( - messages: Message[], - systemPrompt?: string, - tools?: Record[] - ): Promise { - const { systemInstruction, contents } = this.convertMessages(messages, systemPrompt); - - const requestBody: { - contents: Array<{ role: 'user' | 'model'; parts: Array<{ text: string }> }>; - generationConfig: { temperature: number; maxOutputTokens: number }; - systemInstruction?: { parts: Array<{ text: string }> }; - tools?: Record[]; - } = { - contents, - generationConfig: { - temperature: 0.7, - maxOutputTokens: 8192, - } - }; - - if (systemInstruction) { - requestBody.systemInstruction = systemInstruction; - } - - if (tools && tools.length > 0) { - requestBody.tools = tools; - } - - const response = await fetch(`${this.apiEndpoint}:generateContent?key=${this.apiKey}`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(requestBody), - }); - - if (!response.ok) { - throw new Error(`Gemini API error: ${response.status} ${response.statusText}`); - } - - const data = await response.json(); - - // Extract content from Gemini's response format - const candidate = data.candidates?.[0]; - const content = candidate?.content?.parts?.[0]?.text || ''; - - // Extract tool calls if present (Gemini format) - const toolCalls = candidate?.content?.parts - ?.filter((part: { functionCall?: unknown }) => part.functionCall) - ?.map((part: { functionCall: { name: string; args: Record } }) => ({ - id: `tool_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`, // Generate ID - name: part.functionCall.name, - parameters: part.functionCall.args - })) || []; - - return { - content, - toolCalls: toolCalls.length > 0 ? toolCalls : undefined, - finished: candidate?.finishReason === 'STOP' || candidate?.finishReason === 'MAX_TOKENS' - }; - } } \ No newline at end of file diff --git a/src/agent/llm/LLMProvider.ts b/src/agent/llm/LLMProvider.ts index 67d6ee0..71d78f0 100644 --- a/src/agent/llm/LLMProvider.ts +++ b/src/agent/llm/LLMProvider.ts @@ -1,4 +1,4 @@ -import type { StreamChunk, LLMResponse } from './StreamingTypes'; +import type { StreamChunk } from './StreamingTypes'; import type { Message } from '../core/AgentState'; /** @@ -18,16 +18,4 @@ export abstract class LLMProvider { systemPrompt?: string, tools?: Record[] ): AsyncIterableIterator; - - /** - * Generate a complete response from the LLM (non-streaming) - * @param messages The full conversation history with preserved roles - * @param systemPrompt The system prompt (optional, can be included in messages) - * @param tools Available tools (optional for now) - */ - abstract generateCompletion( - messages: Message[], - systemPrompt?: string, - tools?: Record[] - ): Promise; } \ No newline at end of file diff --git a/src/agent/llm/OpenAIProvider.ts b/src/agent/llm/OpenAIProvider.ts index 3e10f0a..0a36ad7 100644 --- a/src/agent/llm/OpenAIProvider.ts +++ b/src/agent/llm/OpenAIProvider.ts @@ -1,8 +1,9 @@ import { LLMProvider } from './LLMProvider'; -import type { StreamChunk, LLMResponse } from './StreamingTypes'; +import type { StreamChunk } from './StreamingTypes'; import type { Message } from '../core/AgentState'; import { ConfigManager } from '../../core/config/ConfigManager'; import { URL_CONSTANTS } from '../../constants/coreConstants'; +import { LLM_PROTOCOL } from '../../constants/llmConstants'; /** * OpenAI API provider implementation @@ -16,6 +17,78 @@ export class OpenAIProvider extends LLMProvider { super(); } + /** + * Build OpenAI-compatible messages array from input messages and system prompt + */ + private buildRequestMessages(messages: Message[], systemPrompt?: string): Array<{ role: string; content: string }> { + const openAIMessages: Array<{ role: string; content: string }> = []; + + // Add system prompt if provided + if (systemPrompt) { + openAIMessages.push({ role: 'system', content: systemPrompt }); + } + + // Add conversation history with preserved roles + openAIMessages.push(...messages.map(msg => ({ + role: msg.role, + content: msg.content + }))); + + return openAIMessages; + } + + /** + * Create API request with proper headers and body + */ + private async createApiRequest(messages: Array<{ role: string; content: string }>, config: ReturnType, streaming: boolean): Promise { + const response = await fetch(config.apiEndpoint, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${config.apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: config.model, + ...(config.flexMode && !config.isCompatibleProvider ? { service_tier: 'flex' } : {}), + messages, + stream: streaming, + }), + }); + + if (!response.ok) { + const errorText = await response.text().catch(() => 'Unknown error'); + throw new Error(`${this.name} API request failed (${response.status}): ${response.statusText}. ${errorText}`); + } + + return response; + } + + /** + * Process thinking and content chunks, yielding appropriate StreamChunks + */ + private async *processContentChunk( + thinking: string | undefined, + content: string | undefined, + lastSegmentType: { current: 'thinking' | 'content' | null } + ): AsyncIterableIterator { + if (typeof thinking === 'string' && thinking.length > 0) { + if (lastSegmentType.current && lastSegmentType.current !== 'thinking') { + yield { type: 'text', content: LLM_PROTOCOL.SEGMENT_SEPARATOR }; + } + yield { type: 'text', content: thinking }; + lastSegmentType.current = 'thinking'; + } + + if (typeof content === 'string' && content.length > 0) { + if (lastSegmentType.current && lastSegmentType.current !== 'content') { + yield { type: 'text', content: LLM_PROTOCOL.SEGMENT_SEPARATOR }; + } + yield { type: 'text', content: content }; + lastSegmentType.current = 'content'; + } + } + + /** * Get current configuration values from ConfigManager */ @@ -50,7 +123,7 @@ export class OpenAIProvider extends LLMProvider { */ private detectStreamFormat(firstChunk: string): boolean { // If it starts with "data: ", it's OpenAI SSE format - if (firstChunk.trim().startsWith('data: ')) { + if (firstChunk.trim().startsWith(LLM_PROTOCOL.SSE_DATA_PREFIX)) { return false; // Not Ollama format } @@ -85,12 +158,12 @@ export class OpenAIProvider extends LLMProvider { * Parse OpenAI's SSE format chunk */ private parseOpenAIChunk(line: string): { thinking?: string; content?: string; isDone?: boolean } { - if (!line.startsWith('data: ')) { + if (!line.startsWith(LLM_PROTOCOL.SSE_DATA_PREFIX)) { return {}; } - const data = line.slice(6); - if (data === '[DONE]') { + const data = line.slice(LLM_PROTOCOL.SSE_DATA_PREFIX.length); + if (data === LLM_PROTOCOL.SSE_DONE_MARKER) { return { isDone: true }; } @@ -109,52 +182,20 @@ export class OpenAIProvider extends LLMProvider { messages: Message[], systemPrompt?: string ): AsyncIterableIterator { - // Get fresh config values const config = this.getCurrentConfig(); - - // Build OpenAI messages array with role preservation - const openAIMessages: Array<{ role: string; content: string }> = []; - - // Add system prompt if provided - if (systemPrompt) { - openAIMessages.push({ role: 'system', content: systemPrompt }); - } - - // Add conversation history with preserved roles - openAIMessages.push(...messages.map(msg => ({ - role: msg.role, - content: msg.content - }))); - - const response = await fetch(config.apiEndpoint, { - method: 'POST', - headers: { - 'Authorization': `Bearer ${config.apiKey}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - model: config.model, - ...(config.flexMode && !config.isCompatibleProvider ? { service_tier: 'flex' } : {}), - messages: openAIMessages, - stream: true, - }), - }); - - if (!response.ok) { - throw new Error(`OpenAI API error: ${response.status} ${response.statusText}`); - } + const requestMessages = this.buildRequestMessages(messages, systemPrompt); + const response = await this.createApiRequest(requestMessages, config, true); const reader = response.body?.getReader(); if (!reader) { - throw new Error('Failed to get response reader'); + throw new Error(`${this.name} streaming: Failed to get response reader from API response`); } const decoder = new TextDecoder(); let buffer = ''; let firstChunkProcessed = false; - let lastSegmentType: 'thinking' | 'content' | null = null; + const lastSegmentType = { current: null as 'thinking' | 'content' | null }; - try { while (true) { const { done, value } = await reader.read(); @@ -177,124 +218,20 @@ export class OpenAIProvider extends LLMProvider { firstChunkProcessed = true; } - if (this.isOllamaFormat) { - const { thinking, content, isDone } = this.parseOllamaChunk(trimmedLine); - if (isDone) { - yield { type: 'done', content: '' }; - return; - } - - if (typeof thinking === 'string' && thinking.length > 0) { - if (lastSegmentType && lastSegmentType !== 'thinking') { - yield { type: 'text', content: '\n\n' }; - } - yield { type: 'text', content: thinking }; - lastSegmentType = 'thinking'; - } - - if (typeof content === 'string' && content.length > 0) { - if (lastSegmentType && lastSegmentType !== 'content') { - yield { type: 'text', content: '\n\n' }; - } - yield { type: 'text', content: content }; - lastSegmentType = 'content'; - } - - continue; - } - - const { thinking, content, isDone } = this.parseOpenAIChunk(trimmedLine); + const { thinking, content, isDone } = this.isOllamaFormat + ? this.parseOllamaChunk(trimmedLine) + : this.parseOpenAIChunk(trimmedLine); + if (isDone) { yield { type: 'done', content: '' }; return; } - - if (typeof thinking === 'string' && thinking.length > 0) { - if (lastSegmentType && lastSegmentType !== 'thinking') { - yield { type: 'text', content: '\n\n' }; - } - yield { type: 'text', content: thinking }; - lastSegmentType = 'thinking'; - } - if (typeof content === 'string' && content.length > 0) { - if (lastSegmentType && lastSegmentType !== 'content') { - yield { type: 'text', content: '\n\n' }; - } - yield { type: 'text', content }; - lastSegmentType = 'content'; - } + yield* this.processContentChunk(thinking, content, lastSegmentType); } } } finally { reader.releaseLock(); } } - - async generateCompletion( - messages: Message[], - systemPrompt?: string - ): Promise { - // Get fresh config values - const config = this.getCurrentConfig(); - - // Build OpenAI messages array with role preservation - const openAIMessages: Array<{ role: string; content: string }> = []; - - // Add system prompt if provided - if (systemPrompt) { - openAIMessages.push({ role: 'system', content: systemPrompt }); - } - - // Add conversation history with preserved roles - openAIMessages.push(...messages.map(msg => ({ - role: msg.role, - content: msg.content - }))); - - const response = await fetch(config.apiEndpoint, { - method: 'POST', - headers: { - 'Authorization': `Bearer ${config.apiKey}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - model: config.model, - ...(config.flexMode && !config.isCompatibleProvider ? { service_tier: 'flex' } : {}), - messages: openAIMessages, - stream: false, - }), - }); - - if (!response.ok) { - throw new Error(`OpenAI API error: ${response.status} ${response.statusText}`); - } - - const data = await response.json(); - - // Handle different response formats - - if (data.message) { - // Ollama/compatible format - const thinking: string = data.message.thinking || ''; - const contentText: string = data.message.content || data.response || ''; - const textCombined = thinking && contentText ? `${thinking}\n\n${contentText}` : (thinking || contentText); - - return { - content: textCombined, - finished: data.done === true || data.done_reason === 'stop' - }; - } else { - // OpenAI format - const choice = data.choices?.[0]; - const thinking: string = choice?.message?.thinking || ''; - const contentText: string = choice?.message?.content || ''; - const textCombined = thinking && contentText ? `${thinking}\n\n${contentText}` : (thinking || contentText); - - return { - content: textCombined, - finished: choice?.finish_reason === 'stop' - }; - } - } } \ No newline at end of file diff --git a/src/constants/llmConstants.ts b/src/constants/llmConstants.ts new file mode 100644 index 0000000..1674081 --- /dev/null +++ b/src/constants/llmConstants.ts @@ -0,0 +1,10 @@ +/** + * Constants related to LLM providers and API interactions + */ + +// Protocol markers and special character combinations used in LLM communications +export const LLM_PROTOCOL = { + SSE_DATA_PREFIX: 'data: ', + SSE_DONE_MARKER: '[DONE]', + SEGMENT_SEPARATOR: '\n\n' +} as const; \ No newline at end of file From 8b0ddae58d9b9b896b9ac5076655299e060f10c5 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Sat, 16 Aug 2025 17:35:40 -0700 Subject: [PATCH 3/7] optimized ChatBox component. --- .gitignore | 1 + src/components/ChatBox.tsx | 373 +++++++------------------------- src/hooks/useStreamProcessor.ts | 119 ++++++++++ src/types/projectTypes.ts | 8 + src/utils/chatMessageUtils.ts | 42 ++++ src/utils/toolExecutionUtils.ts | 102 +++++++++ 6 files changed, 351 insertions(+), 294 deletions(-) create mode 100644 src/hooks/useStreamProcessor.ts create mode 100644 src/utils/chatMessageUtils.ts create mode 100644 src/utils/toolExecutionUtils.ts diff --git a/.gitignore b/.gitignore index dba0c85..f1164d6 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ DEPLOYMENT.md # Editor directories and files !.vscode/extensions.json .idea +.claude .DS_Store *.suo *.ntvs* diff --git a/src/components/ChatBox.tsx b/src/components/ChatBox.tsx index e6a0c97..20b4e2b 100644 --- a/src/components/ChatBox.tsx +++ b/src/components/ChatBox.tsx @@ -8,23 +8,18 @@ import { GeminiProvider } from '../agent/llm/GeminiProvider'; import { LLMProvider } from '../agent/llm/LLMProvider'; import { ConfigManager } from '../core/config/ConfigManager'; import { useProjectStore } from '../stores/projectStore'; -import { XMLToolExecutor } from '../agent/core/XMLToolExecutor'; -import { extractXMLFromString } from '../util/xmlUtil'; import { SystemPrompts } from '../agent/core/SystemPrompts'; import { clearChatHistoryAndUI, registerClearChatUICallback } from '../util/chatUtil'; import { processUserMessage } from '../util/messageFilter/UserMessageFilter'; +import { useStreamProcessor } from '../hooks/useStreamProcessor'; +import { createMessage, addWelcomeMessage } from '../utils/chatMessageUtils'; +import { extractActionableTools, executeAllTools } from '../utils/toolExecutionUtils'; + +import type { ChatMessage } from '../types/projectTypes'; // Module-level guard to avoid duplicate welcome in React StrictMode dev remounts let hasShownWelcomeOnceInRuntime = false; -interface ChatMessage { - id: string; - role: 'user' | 'assistant'; - content: string; - isStreaming?: boolean; - tokenCount?: number; -} - /** * Create the appropriate LLM provider based on configuration */ @@ -55,7 +50,6 @@ const ChatBox: React.FC = ({ isVisible }) => { // Initialize with empty messages const [messages, setMessages] = useState([]); const [isProcessing, setIsProcessing] = useState(false); - const [abortController, setAbortController] = useState(null); const [lastUserMessage, setLastUserMessage] = useState(''); // Tool execution state @@ -66,9 +60,25 @@ const ChatBox: React.FC = ({ isVisible }) => { // Track if this is the first message (for system prompt logging) const [isFirstMessage, setIsFirstMessage] = useState(true); - const generateMessageId = (): string => { - return `msg_${Date.now()}_${Math.random().toString(36).substring(2, 15)}`; - }; + // Message update callbacks for stream processor + const handleMessageUpdate = useCallback((messageId: string, updater: (msg: ChatMessage) => ChatMessage) => { + setMessages(prev => prev.map(msg => msg.id === messageId ? updater(msg) : msg)); + }, []); + + const handleMessageAdd = useCallback((message: ChatMessage) => { + setMessages(prev => [...prev, message]); + }, []); + + const handleProcessingChange = useCallback((processing: boolean) => { + setIsProcessing(processing); + }, []); + + // Stream processor hook + const streamProcessor = useStreamProcessor({ + onMessageUpdate: handleMessageUpdate, + onMessageAdd: handleMessageAdd, + onProcessingChange: handleProcessingChange + }); const clearChatUI = useCallback(async () => { // Clear UI state @@ -78,18 +88,9 @@ const ChatBox: React.FC = ({ isVisible }) => { setIsFirstMessage(true); // Auto-show welcome message after clearing (like on app startup) - try { - const result = await processUserMessage('/welcome'); - if (result.pseudoAssistantResponse) { - const pseudoId = generateMessageId(); - setMessages(prev => [...prev, { - id: pseudoId, - role: 'assistant', - content: result.pseudoAssistantResponse!, - }]); - } - } catch { - // ignore errors, just don't show welcome if it fails + const welcomeMessage = await addWelcomeMessage(); + if (welcomeMessage) { + setMessages([welcomeMessage]); } }, []); @@ -117,29 +118,20 @@ const ChatBox: React.FC = ({ isVisible }) => { // Auto-trigger welcome on first launch (guard against React StrictMode double-invoke only) (async () => { - try { - if (hasShownWelcomeOnceInRuntime) return; - hasShownWelcomeOnceInRuntime = true; + if (hasShownWelcomeOnceInRuntime) return; + hasShownWelcomeOnceInRuntime = true; - const result = await processUserMessage('/welcome'); - if (result.pseudoAssistantResponse) { - const pseudoId = generateMessageId(); - setMessages(prev => [...prev, { - id: pseudoId, - role: 'assistant', - content: result.pseudoAssistantResponse!, - }]); - } - } catch { - // ignore + const welcomeMessage = await addWelcomeMessage(); + if (welcomeMessage) { + setMessages([welcomeMessage]); } })(); }, [clearChatUI]); const handleAbort = () => { - if (abortController) { - abortController.abort(); - setAbortController(null); + const controller = streamProcessor.abortController; + if (controller) { + controller.abort(); // Use AgentCore to clean up the data model and get the user message content const agentCore = AgentCore.instance(); @@ -160,28 +152,9 @@ const ChatBox: React.FC = ({ isVisible }) => { clearChatHistoryAndUI(setStatus); }; - - const addToolResultMessage = (toolName: string, success: boolean, result: string) => { - const toolMsgId = generateMessageId(); - const friendlyDisplay = `${success ? '✅' : '❌'} __**${toolName}**__ \n\n └── ${result}`; - - setMessages(prev => [...prev, { - id: toolMsgId, - role: 'user', - content: friendlyDisplay - }]); - }; - const executeToolsFromResponse = async (response: string): Promise => { try { - // Check if response contains XML tool invocations - const xmlBlocks = extractXMLFromString(response); - // Consider only actionable tools (exclude think/thinking) - const actionableBlocks = xmlBlocks.filter((block) => { - const match = block.match(/<([a-zA-Z_][a-zA-Z0-9_-]*)/); - const name = match ? match[1].toLowerCase() : ''; - return name !== 'think' && name !== 'thinking'; - }); + const actionableBlocks = extractActionableTools(response); if (actionableBlocks.length === 0) { // No actionable tools to execute, stop the loop @@ -194,46 +167,12 @@ const ChatBox: React.FC = ({ isVisible }) => { setCurrentToolIndex(0); const { setStatus } = useProjectStore.getState(); - setStatus(`Executing ${actionableBlocks.length} tool(s)...`); - - const executor = XMLToolExecutor.instance(); - let accumulatedResults = ''; - - // Execute tools sequentially with real-time updates - for (let i = 0; i < actionableBlocks.length; i++) { - setCurrentToolIndex(i + 1); - setStatus(`Executing tool ${i + 1} of ${actionableBlocks.length}...`); - - // Determine tool name from XML block - const toolNameMatch = actionableBlocks[i].match(/<([a-zA-Z_][a-zA-Z0-9_-]*)/); - const toolName = toolNameMatch ? toolNameMatch[1] : 'unknown_tool'; - - try { - // Execute single XML block - const results = await executor.executeXMLTools(actionableBlocks[i]); - const result = results[0]; // Single block should give single result - - if (result) { - // Add friendly display message - addToolResultMessage(toolName, result.success, result.result); - - // Accumulate formatted result for LLM (skip thinking tools) - if (toolName !== 'thinking' && toolName !== 'think') { - const formattedResult = `tool: ${toolName}\nsuccess: ${result.success}\nresult:\n${result.result}\n------------\n`; - accumulatedResults += formattedResult; - } - } - } catch (error) { - // Handle individual tool error - addToolResultMessage(toolName, false, `Tool execution failed: ${error}`); - - // Accumulate error result for LLM (skip thinking tools) - if (toolName !== 'thinking' && toolName !== 'think') { - const formattedResult = `tool: ${toolName}\nsuccess: false\nresult:\nTool execution failed: ${error}\n------------\n`; - accumulatedResults += formattedResult; - } - } - } + + // Execute all tools and get accumulated results + const accumulatedResults = await executeAllTools(actionableBlocks, { + onMessageAdd: handleMessageAdd, + onStatusUpdate: setStatus + }); // Store accumulated results setToolResults(accumulatedResults); @@ -264,90 +203,16 @@ const ChatBox: React.FC = ({ isVisible }) => { }; const sendToolResultsToLLM = async (toolResultsString: string): Promise => { - // Send tool results as hidden user input to LLM - setIsProcessing(true); - - // Create abort controller for this request - const controller = new AbortController(); - setAbortController(controller); - - // Add streaming assistant message for the response - const assistantMsgId = generateMessageId(); - setMessages(prev => [...prev, { - id: assistantMsgId, - role: 'assistant', - content: 'Processing... 0 tokens received. click here to abort.', - isStreaming: true, - tokenCount: 0 - }]); - - try { + // Process tool results through the stream processor + const assistantResponse = await streamProcessor.processStream(toolResultsString, 'TOOL_RESULTS'); + + // Check if the new response contains more tools + const hasMoreTools = await executeToolsFromResponse(assistantResponse); + + // If no more tools were found, set working flag to false + if (!hasMoreTools) { const agentCore = AgentCore.instance(); - let assistantResponse = ''; - let tokenCount = 0; - - // Log the tool results being sent to LLM - console.log('------------ USER ------------'); - console.log(toolResultsString); - console.log('------------------------------'); - - for await (const chunk of agentCore.processUserInput(toolResultsString)) { - // Check if request was aborted - if (controller.signal.aborted) { - return; - } - - if (chunk.type === 'text') { - assistantResponse += chunk.content; - tokenCount++; - - // Update streaming message with token count and abort link - setMessages(prev => prev.map(msg => - msg.id === assistantMsgId - ? { ...msg, content: `Processing... ${tokenCount} tokens received. click here to abort.`, tokenCount } - : msg - )); - } else if (chunk.type === 'done') { - // Replace with final response - setMessages(prev => prev.map(msg => - msg.id === assistantMsgId - ? { ...msg, content: assistantResponse, isStreaming: false, tokenCount: undefined } - : msg - )); - - // Log the complete assistant response - console.log('------------ ASSISTANT ------------'); - console.log(assistantResponse); - console.log('-----------------------------------'); - - // Check if the new response contains more tools - const hasMoreTools = await executeToolsFromResponse(assistantResponse); - - // If no more tools were found, set working flag to false - if (!hasMoreTools) { - const agentCore = AgentCore.instance(); - agentCore.getAgentState().setIsWorkingOnTask(false); - } - - break; - } - } - } catch (error) { - if (error instanceof Error && error.name === 'AbortError') { - // Request was aborted, don't show error - return; - } - - console.error('Error processing tool results:', error); - // Update with error message - setMessages(prev => prev.map(msg => - msg.id === assistantMsgId - ? { ...msg, content: 'Error: Failed to process tool results', isStreaming: false, tokenCount: undefined } - : msg - )); - } finally { - setAbortController(null); - setIsProcessing(false); + agentCore.getAgentState().setIsWorkingOnTask(false); } }; @@ -362,22 +227,14 @@ const ChatBox: React.FC = ({ isVisible }) => { // Conditionally show the user message bubble if (filterResult.displayUserMessage) { - const userMsgId = generateMessageId(); - setMessages(prev => [...prev, { - id: userMsgId, - role: 'user', - content: userMessage - }]); + const userMsgObject = createMessage('user', userMessage); + handleMessageAdd(userMsgObject); } // If we have a pseudo assistant response, show it immediately if (filterResult.pseudoAssistantResponse) { - const pseudoId = generateMessageId(); - setMessages(prev => [...prev, { - id: pseudoId, - role: 'assistant', - content: filterResult.pseudoAssistantResponse!, - }]); + const pseudoMessage = createMessage('assistant', filterResult.pseudoAssistantResponse); + handleMessageAdd(pseudoMessage); } // If we shouldn't send anything to LLM, stop here @@ -385,105 +242,33 @@ const ChatBox: React.FC = ({ isVisible }) => { return; } - setIsProcessing(true); + // Set working on task flag when user sends a message + const agentCore = AgentCore.instance(); + agentCore.getAgentState().setIsWorkingOnTask(true); - // Create abort controller for this request - const controller = new AbortController(); - setAbortController(controller); - - // Add streaming assistant message - const assistantMsgId = generateMessageId(); - setMessages(prev => [...prev, { - id: assistantMsgId, - role: 'assistant', - content: 'Processing... 0 tokens received. click here to abort.', - isStreaming: true, - tokenCount: 0 - }]); - - try { - const agentCore = AgentCore.instance(); - let assistantResponse = ''; - let tokenCount = 0; - - // Set working on task flag when user sends a message - agentCore.getAgentState().setIsWorkingOnTask(true); - - // Log system prompt only for first message or first message after clear - if (isFirstMessage) { - try { - const systemPrompt = await SystemPrompts.getSystemPromptWithContext(); - console.log('------------ SYSTEM ------------'); - console.log(systemPrompt); - console.log('--------------------------------'); - } catch (error) { - console.error('Failed to log system prompt:', error); - } - // Mark that we've logged the system prompt for this conversation - setIsFirstMessage(false); + // Log system prompt only for first message or first message after clear + if (isFirstMessage) { + try { + const systemPrompt = await SystemPrompts.getSystemPromptWithContext(); + console.log('------------ SYSTEM ------------'); + console.log(systemPrompt); + console.log('--------------------------------'); + } catch (error) { + console.error('Failed to log system prompt:', error); } + // Mark that we've logged the system prompt for this conversation + setIsFirstMessage(false); + } - // Log the final user message being sent to LLM - console.log('------------ USER ------------'); - console.log(filterResult.finalMessageForLLM); - console.log('------------------------------'); - - for await (const chunk of agentCore.processUserInput(filterResult.finalMessageForLLM)) { - // Check if request was aborted - if (controller.signal.aborted) { - return; - } - - if (chunk.type === 'text') { - assistantResponse += chunk.content; - tokenCount++; - - // Update streaming message with token count and abort link - setMessages(prev => prev.map(msg => - msg.id === assistantMsgId - ? { ...msg, content: `Processing... ${tokenCount} tokens received. click here to abort.`, tokenCount } - : msg - )); - } else if (chunk.type === 'done') { - // Replace with final response - setMessages(prev => prev.map(msg => - msg.id === assistantMsgId - ? { ...msg, content: assistantResponse, isStreaming: false, tokenCount: undefined } - : msg - )); - - // Log the complete assistant response - console.log('------------ ASSISTANT ------------'); - console.log(assistantResponse); - console.log('-----------------------------------'); - - // Check if response contains tools to execute - const hasTools = await executeToolsFromResponse(assistantResponse); - - // If no tools were found, set working flag to false and return control to user - if (!hasTools) { - agentCore.getAgentState().setIsWorkingOnTask(false); - } - - break; - } - } - } catch (error) { - if (error instanceof Error && error.name === 'AbortError') { - // Request was aborted, don't show error - return; - } - - console.error('Error processing message:', error); - // Update with error message - setMessages(prev => prev.map(msg => - msg.id === assistantMsgId - ? { ...msg, content: 'Error: Failed to process message', isStreaming: false, tokenCount: undefined } - : msg - )); - } finally { - setAbortController(null); - setIsProcessing(false); + // Process user input through the stream processor + const assistantResponse = await streamProcessor.processStream(filterResult.finalMessageForLLM, 'USER'); + + // Check if response contains tools to execute + const hasTools = await executeToolsFromResponse(assistantResponse); + + // If no tools were found, set working flag to false and return control to user + if (!hasTools) { + agentCore.getAgentState().setIsWorkingOnTask(false); } } }; diff --git a/src/hooks/useStreamProcessor.ts b/src/hooks/useStreamProcessor.ts new file mode 100644 index 0000000..c04c74e --- /dev/null +++ b/src/hooks/useStreamProcessor.ts @@ -0,0 +1,119 @@ +import { useState, useCallback } from 'react'; +import { AgentCore } from '../agent/core/AgentCore'; +import { createStreamingMessage } from '../utils/chatMessageUtils'; +import type { ChatMessage } from '../types/projectTypes'; + +interface StreamProcessorOptions { + onMessageUpdate: (messageId: string, updater: (msg: ChatMessage) => ChatMessage) => void; + onMessageAdd: (message: ChatMessage) => void; + onProcessingChange: (isProcessing: boolean) => void; +} + +interface StreamProcessorResult { + processStream: (input: string, logPrefix?: string) => Promise; + abortController: AbortController | null; + isProcessing: boolean; +} + +export const useStreamProcessor = (options: StreamProcessorOptions): StreamProcessorResult => { + const { onMessageUpdate, onMessageAdd, onProcessingChange } = options; + const [abortController, setAbortController] = useState(null); + const [isProcessing, setIsProcessing] = useState(false); + + const processStream = useCallback(async (input: string, logPrefix: string = 'USER'): Promise => { + setIsProcessing(true); + onProcessingChange(true); + + // Create abort controller for this request + const controller = new AbortController(); + setAbortController(controller); + + // Add streaming assistant message + const streamingMessage = createStreamingMessage(); + onMessageAdd(streamingMessage); + + try { + const agentCore = AgentCore.instance(); + let assistantResponse = ''; + let tokenCount = 0; + let streamCompleted = false; + + // Log the input being sent to LLM + console.log(`------------ ${logPrefix} ------------`); + console.log(input); + console.log('------------------------------'); + + for await (const chunk of agentCore.processUserInput(input)) { + // Check if request was aborted + if (controller.signal.aborted) { + return ''; + } + + if (chunk.type === 'text') { + assistantResponse += chunk.content; + tokenCount++; + + // Update streaming message with token count and abort link + onMessageUpdate(streamingMessage.id, (msg) => ({ + ...msg, + content: `Processing... ${tokenCount} tokens received. click here to abort.`, + tokenCount + })); + } else if (chunk.type === 'done') { + streamCompleted = true; + // Replace with final response + onMessageUpdate(streamingMessage.id, (msg) => ({ + ...msg, + content: assistantResponse, + isStreaming: false, + tokenCount: undefined + })); + + // Log the complete assistant response + console.log('------------ ASSISTANT ------------'); + console.log(assistantResponse); + console.log('-----------------------------------'); + + break; + } + } + + // If stream didn't complete normally, finalize the message + if (!streamCompleted && !controller.signal.aborted) { + onMessageUpdate(streamingMessage.id, (msg) => ({ + ...msg, + content: assistantResponse || 'Stream was interrupted unexpectedly', + isStreaming: false, + tokenCount: undefined + })); + } + + return assistantResponse; + } catch (error) { + if (error instanceof Error && error.name === 'AbortError') { + // Request was aborted, don't show error + return ''; + } + + console.error('Error processing stream:', error); + // Update with error message + onMessageUpdate(streamingMessage.id, (msg) => ({ + ...msg, + content: 'Error: Failed to process message', + isStreaming: false, + tokenCount: undefined + })); + return ''; + } finally { + setAbortController(null); + setIsProcessing(false); + onProcessingChange(false); + } + }, [onMessageUpdate, onMessageAdd, onProcessingChange]); + + return { + processStream, + abortController, + isProcessing + }; +}; \ No newline at end of file diff --git a/src/types/projectTypes.ts b/src/types/projectTypes.ts index 98d4e27..3c59d53 100644 --- a/src/types/projectTypes.ts +++ b/src/types/projectTypes.ts @@ -5,6 +5,14 @@ export interface TimeSignature { denominator: number; } +export interface ChatMessage { + id: string; + role: 'user' | 'assistant'; + content: string; + isStreaming?: boolean; + tokenCount?: number; +} + /** * A reusable class-transformer decorator to apply a default value during deserialization. * @param defaultValue The default value to apply if the field is undefined. diff --git a/src/utils/chatMessageUtils.ts b/src/utils/chatMessageUtils.ts new file mode 100644 index 0000000..393f05a --- /dev/null +++ b/src/utils/chatMessageUtils.ts @@ -0,0 +1,42 @@ +import { processUserMessage } from '../util/messageFilter/UserMessageFilter'; +import type { ChatMessage } from '../types/projectTypes'; + +export const generateMessageId = (): string => { + return `msg_${Date.now()}_${Math.random().toString(36).substring(2, 15)}`; +}; + +export const createMessage = (role: 'user' | 'assistant', content: string): ChatMessage => { + return { + id: generateMessageId(), + role, + content, + }; +}; + +export const createStreamingMessage = (content: string = 'Processing... 0 tokens received. click here to abort.'): ChatMessage => { + return { + id: generateMessageId(), + role: 'assistant', + content, + isStreaming: true, + tokenCount: 0, + }; +}; + +export const createToolResultMessage = (toolName: string, success: boolean, result: string): ChatMessage => { + const friendlyDisplay = `${success ? '✅' : '❌'} __**${toolName}**__ \n\n └── ${result}`; + return createMessage('user', friendlyDisplay); +}; + +export const addWelcomeMessage = async (): Promise => { + try { + const result = await processUserMessage('/welcome'); + if (result.pseudoAssistantResponse) { + return createMessage('assistant', result.pseudoAssistantResponse); + } + return null; + } catch { + // ignore errors, just don't show welcome if it fails + return null; + } +}; \ No newline at end of file diff --git a/src/utils/toolExecutionUtils.ts b/src/utils/toolExecutionUtils.ts new file mode 100644 index 0000000..63a4dfa --- /dev/null +++ b/src/utils/toolExecutionUtils.ts @@ -0,0 +1,102 @@ +import { XMLToolExecutor } from '../agent/core/XMLToolExecutor'; +import { extractXMLFromString } from '../util/xmlUtil'; +import { createToolResultMessage } from './chatMessageUtils'; +import type { ChatMessage } from '../types/projectTypes'; + +interface ToolExecutionResult { + success: boolean; + result: string; +} + +interface ToolExecutionOptions { + onMessageAdd: (message: ChatMessage) => void; + onStatusUpdate: (status: string) => void; +} + +export const extractActionableTools = (response: string): string[] => { + const xmlBlocks = extractXMLFromString(response); + // Consider only actionable tools (exclude think/thinking) + return xmlBlocks.filter((block) => { + const match = block.match(/<([a-zA-Z_][a-zA-Z0-9_-]*)/); + const name = match ? match[1].toLowerCase() : ''; + return name !== 'think' && name !== 'thinking'; + }); +}; + +export const extractToolName = (xmlBlock: string): string => { + const toolNameMatch = xmlBlock.match(/<([a-zA-Z_][a-zA-Z0-9_-]*)/); + return toolNameMatch ? toolNameMatch[1] : 'unknown_tool'; +}; + +export const executeSingleTool = async ( + xmlBlock: string, + toolName: string, + options: ToolExecutionOptions +): Promise => { + const { onMessageAdd } = options; + + try { + const executor = XMLToolExecutor.instance(); + const results = await executor.executeXMLTools(xmlBlock); + const result = results[0]; // Single block should give single result + + if (result) { + // Add friendly display message + const toolMessage = createToolResultMessage(toolName, result.success, result.result); + onMessageAdd(toolMessage); + + return { + success: result.success, + result: result.result + }; + } + + return { + success: false, + result: 'No result returned from tool execution' + }; + } catch (error) { + // Handle individual tool error + const errorMessage = `Tool execution failed: ${error}`; + const toolMessage = createToolResultMessage(toolName, false, errorMessage); + onMessageAdd(toolMessage); + + return { + success: false, + result: errorMessage + }; + } +}; + +export const formatToolResultForLLM = (toolName: string, result: ToolExecutionResult): string => { + // Skip thinking tools + if (toolName === 'thinking' || toolName === 'think') { + return ''; + } + + return `tool: ${toolName}\nsuccess: ${result.success}\nresult:\n${result.result}\n------------\n`; +}; + +export const executeAllTools = async ( + actionableBlocks: string[], + options: ToolExecutionOptions +): Promise => { + const { onStatusUpdate } = options; + + onStatusUpdate(`Executing ${actionableBlocks.length} tool(s)...`); + + let accumulatedResults = ''; + + // Execute tools sequentially with real-time updates + for (let i = 0; i < actionableBlocks.length; i++) { + onStatusUpdate(`Executing tool ${i + 1} of ${actionableBlocks.length}...`); + + const toolName = extractToolName(actionableBlocks[i]); + const result = await executeSingleTool(actionableBlocks[i], toolName, options); + + // Accumulate formatted result for LLM + accumulatedResults += formatToolResultForLLM(toolName, result); + } + + return accumulatedResults; +}; \ No newline at end of file From 365acdae7e489b19072b9a23d5f814df86b53eea Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Sat, 16 Aug 2025 17:45:28 -0700 Subject: [PATCH 4/7] added overview document. --- docs/technical/overview.md | 600 +++++++++++++++++++++++++++++++++++++ 1 file changed, 600 insertions(+) create mode 100644 docs/technical/overview.md diff --git a/docs/technical/overview.md b/docs/technical/overview.md new file mode 100644 index 0000000..2a3eb3c --- /dev/null +++ b/docs/technical/overview.md @@ -0,0 +1,600 @@ +# KGStudio: Digital Audio Workstation (DAW) Project Overview + +## Project Introduction + +KGStudio is a light-weighted, modern, web-based Digital Audio Workstation (DAW) built with React, TypeScript, and Zustand. The project aims to provide a professional-grade music production environment in the browser, with features comparable to desktop DAWs like Ableton Live, FL Studio, or Logic Pro. + +## Ultimate Goals + +1. Create a fully-functional DAW that runs in modern web browsers +2. Provide a professional-grade UI with intuitive workflows for music production +3. Support MIDI and audio recording, editing, and playback +4. Implement a plugin system for virtual instruments and effects +5. Enable project saving, loading, and export functionality +6. Optimize for performance to handle complex projects with many tracks + +## Current Tech Stack + +- **Frontend Framework**: React with TypeScript +- **State Management**: Zustand +- **Build Tool**: Vite +- **UI Components**: Custom components with CSS; icons via React Icons +- **Audio Engine**: Tone.js for Web Audio synthesis and playback (real soundfonts via Sampler) +- **Data Persistence**: IndexedDB using `idb`, class serialization with `class-transformer` +- **AI/Agent**: Configurable LLM provider (OpenAI/Claude/Gemini/compatible) with XML tool execution +- **Architecture Pattern**: Core/UI separation with a domain model and component-based UI + +## Project Structure + +``` +KGStudio/ +├── docs/ # Project documentation +│ ├── USER_GUIDE.md # End-user guide +│ └── technical/ +│ └── overview.md # Technical project overview (this document) +├── public/ # Static assets +│ ├── apple-touch-icon.png # PWA icon (Apple) +│ ├── config.json # Default application configuration +│ ├── favicon-96x96.png # Favicon (96x96) +│ ├── favicon.ico # Favicon (ICO) +│ ├── favicon.svg # Favicon (SVG) +│ ├── logo.png # Application logo +│ ├── logo-kgaudiolab.png # Alternate brand logo (KGAudioLab) +│ ├── site.webmanifest # PWA manifest +│ ├── web-app-manifest-192x192.png # PWA icon (192x192) +│ ├── web-app-manifest-512x512.png # PWA icon (512x512) +│ ├── vite.svg # Vite logo +│ ├── chat/ # Chat UI copy and errors +│ │ ├── error_no_openai_compatible_base_url.md +│ │ ├── error_no_openai_compatible_model.md +│ │ ├── error_no_openai_key.md +│ │ ├── error_no_selected_region.md +│ │ ├── welcome_again.md +│ │ ├── welcome_new.md +│ │ ├── custom_instructions_gpt-4o.md # Custom instructions template for GPT-4o +│ │ └── custom_instructions_qwen3-a3b-30b.md # Custom instructions template for Qwen3-A3B-30B +│ ├── demo/ # Demo assets +│ │ ├── cover-FXgihfAH2vc.png # Demo cover image +│ │ └── cover-vKbWAQRt0r0.png # Demo cover image +│ ├── prompts/ # AI system prompts +│ │ ├── system.md +│ │ ├── system_20250806.md +│ │ ├── user_msg_appendix.md +│ │ └── README.md # Prompt set overview +│ └── resources/ # Application resources +│ ├── icon.png +│ ├── instrument_bg.png +│ ├── instrument_bg_v0.png +│ └── instruments/ # Instrument icons (General MIDI) +│ ├── bass.png +│ ├── brass_ensemble.png +│ ├── cello.png +│ ├── clarinet.png +│ ├── contrabass.png +│ ├── drawbar_organ.png +│ ├── drums.png +│ ├── electric_guitar.png +│ ├── electric_piano.png +│ ├── flute.png +│ ├── french_horn.png +│ ├── guitar.png +│ ├── harp.png +│ ├── oboe.png +│ ├── orchestra_percussion_kit.png +│ ├── piano.png +│ ├── sax.png +│ ├── string_ensemble.png +│ ├── synth.png +│ ├── trombone.png +│ ├── trumpet.png +│ ├── viola.png +│ └── violin.png +├── src/ +│ ├── agent/ # AI agent system +│ │ ├── core/ # Agent core components +│ │ │ ├── AgentCore.ts # Main agent orchestration +│ │ │ ├── AgentState.ts # Agent state management +│ │ │ ├── SystemPrompts.ts # System prompts for AI +│ │ │ └── XMLToolExecutor.ts # XML tool execution engine +│ │ ├── llm/ # LLM integration +│ │ │ ├── ClaudeProvider.ts +│ │ │ ├── GeminiProvider.ts +│ │ │ ├── LLMProvider.ts # Abstract LLM provider interface +│ │ │ ├── OpenAIProvider.ts # OpenAI API integration +│ │ │ └── StreamingTypes.ts # Streaming response types +│ │ └── tools/ # Agent tools +│ │ ├── AddNotesTool.ts # Tool for adding notes +│ │ ├── AttemptCompletionTool.ts # Mark current task as completed +│ │ ├── BaseTool.ts # Base tool class +│ │ ├── ReadMusicTool.ts # Tool for reading music +│ │ ├── RemoveNotesTool.ts # Tool for removing notes +│ │ ├── ThinkTool.ts # Background thinking tool +│ │ ├── ThinkingTool.ts # Alternative thought tool +│ │ └── index.ts # Tool exports +│ ├── App.css # Main application styles +│ ├── App.tsx # Main application component +│ ├── assets/ +│ │ └── react.svg # React logo +│ ├── components/ # React UI components +│ │ ├── ChatBox.tsx # Chatbox component +│ │ ├── InstrumentSelection.tsx # Instrument picker +│ │ ├── TrackControl.tsx # Track control component +│ │ ├── chat/ +│ │ │ ├── AssistantMessage.tsx +│ │ │ ├── UserMessage.tsx +│ │ │ └── index.ts +│ │ ├── common/ # Common reusable components +│ │ │ ├── FileImportModal.tsx +│ │ │ ├── KGDropdown.tsx +│ │ │ ├── LoadingOverlay.tsx +│ │ │ ├── Playhead.tsx +│ │ │ ├── icons/ +│ │ │ │ └── PianoIcon.tsx +│ │ │ └── index.ts +│ │ ├── interfaces.ts # Shared interfaces +│ │ ├── MainContent.tsx # Main track display area +│ │ ├── piano-roll/ # Piano roll related components +│ │ │ ├── PianoGrid.tsx +│ │ │ ├── PianoGridHeader.tsx +│ │ │ ├── PianoKeys.tsx +│ │ │ ├── PianoNote.tsx +│ │ │ ├── PianoRoll.tsx +│ │ │ ├── PianoRollContent.tsx +│ │ │ ├── PianoRollHeader.tsx +│ │ │ ├── PianoRollToolbar.tsx +│ │ │ └── SelectionBox.tsx +│ │ ├── settings/ +│ │ │ ├── SettingsPanel.tsx +│ │ │ ├── SettingsSidebar.tsx +│ │ │ ├── index.ts +│ │ │ └── sections/ +│ │ │ ├── BehaviorSettings.tsx +│ │ │ ├── GeneralSettings.tsx +│ │ │ └── TemplatesSettings.tsx +│ │ ├── StatusBar.tsx # Status bar component +│ │ ├── Toolbar.tsx # Top toolbar component +│ │ └── track/ +│ │ ├── RegionItem.tsx +│ │ ├── TrackGridItem.tsx +│ │ ├── TrackGridPanel.tsx +│ │ ├── TrackInfoItem.tsx +│ │ └── TrackInfoPanel.tsx +│ ├── constants/ # Application constants +│ │ ├── coreConstants.ts # Core application constants (DB, audio, etc.) +│ │ ├── generalMidiConstants.ts # General MIDI mapping/constants +│ │ ├── index.ts # Constants re-export +│ │ ├── midiConstants.ts # MIDI message constants +│ │ └── uiConstants.ts # UI-related constants +│ ├── core/ # Core application logic +│ │ ├── KGCore.ts # Main application singleton +│ │ ├── KGDebugger.ts # Debug utilities and testing tools +│ │ ├── KGProject.ts # Project model +│ │ ├── audio-interface/ # Advanced audio engine integration +│ │ │ ├── KGAudioBus.ts # Individual track audio bus +│ │ │ ├── KGAudioInterface.ts # Audio engine coordinator +│ │ │ ├── KGToneBuffersPool.ts # Soundfont buffers pool +│ │ │ └── KGToneSamplerFactory.ts # Sampler factory +│ │ ├── commands/ # Command pattern implementation for undo/redo +│ │ │ ├── KGCommand.ts # Base command class +│ │ │ ├── KGCommandHistory.ts # Command history manager +│ │ │ ├── index.ts # Command exports +│ │ │ ├── note/ +│ │ │ │ ├── CreateNoteCommand.ts +│ │ │ │ ├── CreateNotesCommand.ts +│ │ │ │ ├── DeleteNotesCommand.ts +│ │ │ │ ├── MoveNotesCommand.ts +│ │ │ │ ├── PasteNotesCommand.ts +│ │ │ │ └── ResizeNotesCommand.ts +│ │ │ ├── project/ +│ │ │ │ └── ChangeProjectPropertyCommand.ts +│ │ │ ├── track/ +│ │ │ │ ├── AddTrackCommand.ts # Track command: add track +│ │ │ │ ├── RemoveTrackCommand.ts # Track command: remove track +│ │ │ │ ├── ReorderTracksCommand.ts # Track command: reorder tracks +│ │ │ │ └── UpdateTrackCommand.ts # Track command: update track +│ │ │ └── region/ +│ │ │ ├── CreateRegionCommand.ts +│ │ │ ├── DeleteRegionCommand.ts +│ │ │ ├── MoveRegionCommand.ts +│ │ │ ├── PasteRegionsCommand.ts +│ │ │ ├── ResizeRegionCommand.ts +│ │ │ └── UpdateRegionCommand.ts +│ │ ├── config/ +│ │ │ ├── ConfigManager.ts # JSON config loading/persistence +│ │ │ └── index.ts +│ │ ├── io/ +│ │ │ └── KGStorage.ts # IndexedDB storage layer +│ │ ├── midi/ +│ │ │ └── KGMidiNote.ts +│ │ ├── project-upgrader/ +│ │ │ ├── KGProjectUpgrader.ts +│ │ │ └── upgradeToV1.ts +│ │ ├── region/ +│ │ │ ├── KGMidiRegion.ts +│ │ │ └── KGRegion.ts +│ │ ├── state/ +│ │ │ ├── KGMainContentState.ts +│ │ │ └── KGPianoRollState.ts +│ │ └── track/ +│ │ ├── KGMidiTrack.ts +│ │ └── KGTrack.ts +│ ├── hooks/ +│ │ ├── useConfig.ts +│ │ ├── useGlobalKeyboardHandler.ts +│ │ ├── useNoteOperations.ts +│ │ ├── useNoteSelection.ts +│ │ └── useRegionOperations.ts +│ ├── index.css # Global styles +│ ├── main.tsx # Application entry point +│ ├── mock/ +│ │ └── mockChat.ts # Mock chat data for testing +│ ├── stores/ +│ │ └── projectStore.ts # Project state management (Zustand) +│ ├── types/ +│ │ └── projectTypes.ts # TypeScript types +│ └── util/ # Utility functions +│ ├── abcNotationUtil.ts +│ ├── chatUtil.ts +│ ├── copyPasteUtil.ts +│ ├── mathUtil.ts +│ ├── messageFilter/ +│ │ └── UserMessageFilter.ts +│ ├── midiUtil.ts +│ ├── miscUtil.ts +│ ├── osUtil.ts +│ ├── regionDeleteUtil.ts +│ ├── saveUtil.ts +│ ├── timeUtil.ts +│ └── xmlUtil.ts +│ +│ └── vite-env.d.ts # Vite environment types +├── CLAUDE.md # Claude-specific documentation +├── LICENSE # Project license +├── eslint.config.js # ESLint configuration +├── index.html # HTML entry point +├── package-lock.json # NPM lock file +├── package.json # NPM dependencies and scripts +├── README.md # Project readme +├── tsconfig.app.json # App-specific TypeScript config +├── tsconfig.json # TypeScript configuration +├── tsconfig.node.json # Node-specific TypeScript config +├── vite.config.ts # Vite build configuration +└── .gitignore # Git ignore rules +``` + +## Current Implementation + +### Core Architecture + +The application follows a core/UI separation pattern: + +1. **KGCore** (Singleton): The main application class that manages the audio engine, project state, command history, and provides a global access point. +2. **Command System**: Complete undo/redo implementation using the Command Pattern: + - **KGCommand**: Abstract base class for all undoable operations + - **KGCommandHistory**: Manages command history with undo/redo stack and memory limits + - **Track Commands**: Add, remove, reorder, and update tracks with full undo support + - **Region Commands**: Create, delete, resize, move, paste, and update regions with undo support + - **Note Commands**: Create, delete, resize, move, and paste notes with undo support + - **Project Commands**: Change project properties (name, BPM, time signature) with selective undo +3. **ConfigManager** (Singleton): Manages application configuration including hotkeys, general settings (LLM provider, API keys, soundfont base URL), behavior (e.g., chatbox default open), and templates. Loads defaults from `/public/config.json` (with robust fallback) and persists user customizations via KGStorage. +4. **KGStorage** (Singleton): Generic storage system providing unified access to IndexedDB for projects, configuration, and other data. Replaces individual storage implementations with a centralized, reusable storage layer. +5. **KGAudioInterface** (Singleton): High-level audio engine coordinator that manages track audio buses and orchestrates playback. +6. **KGAudioBus**: Individual track audio processing unit with realistic instrument samples, volume, mute, solo, and effect chain support. +7. **KGToneSamplerFactory** (Singleton): Factory for creating Tone.js samplers loaded with high-quality soundfont samples. +8. **KGToneBuffersPool** (Singleton): Efficient buffer management system that loads and caches soundfont audio data from remote CDNs. +9. **KGProject**: Represents a music project with properties like name, BPM, time signature, and tracks. +10. **KGTrack/KGMidiTrack**: Represents a track in the project with instrument support for MIDI tracks. +11. **Project Upgrader**: On load/import, `upgradeProjectToLatest` migrates legacy projects to the latest `KGProject.CURRENT_PROJECT_STRUCTURE_VERSION` (e.g., instrument mapping in V1). +11. **KGRegion/KGMidiRegion**: Represents a region in a track (a MIDI clip, audio clip, etc.). +12. **KGMidiNote**: Represents a MIDI note with properties like pitch, velocity, start and end beats. + +### Component Architecture + +The UI follows a hierarchical component-based architecture: + +1. **App**: Main container component that orchestrates the overall application layout +2. **Toolbar**: Handles top toolbar functionality including transport controls and project name +3. **MainContent**: Coordinates between track info and grid panels, manages data flow + - **TrackInfoPanel**: Container for track information panels + - **TrackInfoItem**: Individual track information panel with controls + - **TrackGridPanel**: Container for track grid areas + - **TrackGridItem**: Individual track grid with regions + - **RegionItem**: Individual region within a track +4. **TrackControl**: Provides controls for adding and managing tracks +5. **StatusBar**: Displays application status information +6. **PianoRoll**: Modal component for MIDI note editing + - **PianoRollHeader**: Header component with title and close button + - **PianoRollToolbar**: Toolbar with editing tools and quantization options + - **PianoRollContent**: Main content area that orchestrates piano roll components + - **PianoGridHeader**: Bar numbers display at the top of the grid + - **PianoKeys**: Piano keyboard visualization on the left side + - **PianoGrid**: Main grid area for note editing + - **PianoNote**: Individual MIDI note component + - **SelectionBox**: Box selection UI for selecting multiple notes +7. **Common Components**: Reusable UI components + - **KGDropdown**: Reusable dropdown component for consistent dropdown behavior + +### Settings System + +- Dedicated settings UI with sections: `General`, `Behavior`, `Templates`. +- Uses `ConfigManager` and `useConfig` hook for auto-load and debounced save. +- General: LLM provider/model/keys, OpenAI-compatible base URL, soundfont base URL. +- Behavior: chatbox default open at startup. +- Templates: custom instructions for the agent. + +### AI Agent & Chat Integration + +- `ChatBox` integrates an `AgentCore` with pluggable LLM providers (OpenAI, Claude, Gemini, OpenAI-compatible) selected via settings. +- User messages pass through `UserMessageFilter` supporting slash-commands: `/clear`, `/welcome` and context validation (e.g., require a selected region). +- Agent responses can include XML tool calls executed by `XMLToolExecutor` with tools such as `AddNotesTool`, `RemoveNotesTool`, `ReadMusicTool`, `ThinkTool`. +- System and user prompt appendix are loaded from `/public/prompts` for richer context. + +### Instrument & Soundfont System + +- Instrument selection panel with groups and previews using `FLUIDR3_INSTRUMENT_MAP` and General MIDI groupings. +- Realistic playback via Tone.Sampler created by `KGToneSamplerFactory` and buffers from `KGToneBuffersPool` (downloaded from a configurable CDN). +- Global loading overlay shows while instrument buffers are loading; auto-hides and warns if loading takes too long. + +### Keyboard Shortcuts + +- Shortcuts are configurable via `config.json` and `ConfigManager`. +- Global: play/pause, undo/redo, copy/cut/paste, save, hold-to-create-region. +- Piano roll: tool switching (select/pencil), hold-to-create-note, snapping presets and quantize position/length presets. + +This structure provides clear separation of concerns and improves maintainability. + +### Custom Hooks + +The application uses custom hooks to extract and reuse complex logic: + +1. **useNoteOperations**: Manages note creation, resizing, and dragging operations +2. **useNoteSelection**: Manages note selection, including individual selection and box selection +3. **useGlobalKeyboardHandler**: Handles global keyboard shortcuts for copy/paste functionality across the application + +### State Management + +Zustand is used for state management with a primary store, complemented by singleton state classes: + +1. **projectStore**: Manages the project state including project name, tracks, BPM, time signature, and provides actions for modifying the project. +2. **KGPianoRollState**: Singleton state management for piano roll settings including active tool, snapping options, and quantization preferences. + +### Current Features + +1. **Project Management**: + - Create and load projects + - Set project properties (name, BPM, time signature) + - **Project persistence**: Save and load projects using IndexedDB with automatic serialization + - **File operations**: New, Load, Save, and Export buttons in toolbar with confirmation dialogs + - **Automatic data model serialization**: All project data (tracks, regions, notes) automatically preserved + +2. **Track Management**: + - Add tracks with realistic instrument selection (Piano, Guitar, Bass, Drums) + - Rename tracks + - Reorder tracks via drag and drop + - Advanced track controls (volume, solo, mute) with real-time audio processing + - **Modular Audio Bus Architecture**: Each track gets a dedicated KGAudioBus with realistic instrument samples + - **Async Track Creation**: Tracks load high-quality soundfont samples on creation + - **Instrument Switching**: Change track instruments with seamless audio transitions + +3. **Region Management**: + - Create regions by double-clicking on track grid + - Regions display with headers and content areas + - Regions are tied to the data model (KGMidiRegion) + - Regions maintain proper positioning when tracks are reordered + - Resize regions from both start and end edges with bar snapping + - **Expand from beginning**: Left edge resize properly adjusts note positions to maintain absolute timing + - Move regions horizontally within tracks and vertically between tracks via drag and drop + - Visual feedback during resize and drag operations (cursor changes, animation effects) + - **Canvas-based note visualization**: Real-time display of MIDI notes as white horizontal lines within regions + - **Dynamic pitch centering**: Automatically centers display around note content or C4 as fallback + - **Adaptive pitch spacing**: Compresses note spacing when range is large to ensure all notes are visible + +4. **Piano Roll**: + - Draggable/resizable panel + - Piano keyboard visualization + - Grid visualization for note editing + - Auto-scroll to middle C + - Toolbar with editing tools (pointer, pencil) + - Snapping options with triplet support for note positioning during drag operations + - Quantization options for note position and length with triplet support + - Keyboard shortcut (ESC) to close the piano roll + - Rename dialog prevention during drag operations + - Note creation with double-click + - Note resizing from both edges with minimum length constraint + - Note dragging with horizontal snapping and vertical movement + - Multi-note operations: resize and drag deltas applied to all selected notes + - Visual feedback during resize and drag operations + - Note selection with click and shift+click for multi-selection + - Box selection with shift+drag toggle behavior for multi-selection + - Quantize position: snap selected notes to nearest beat grid based on time signature + - Quantize length: adjust note durations with smart extension for short notes + - Integration with core selection system and singleton state management + - **Note Preview with Real Instruments**: Immediate audio feedback when creating notes using authentic instrument samples + +5. **Configuration & Storage Management**: + - **JSON-based configuration**: Default settings loaded from `/public/config.json` including hotkeys and general preferences + - **User customization persistence**: ConfigManager maintains user overrides while preserving defaults + - **Generic storage layer**: KGStorage provides unified IndexedDB access for all data types (projects, config, etc.) + - **Centralized constants**: All database and storage constants managed in `coreConstants.ts` + - **Type-safe configuration**: Full TypeScript interfaces for configuration structure validation + +6. **Data Persistence**: + - **Automatic serialization/deserialization**: Uses class-transformer for seamless data conversion + - **Class instance preservation**: All objects maintain their methods and inheritance after save/load + - **Browser-based storage**: Projects stored locally using IndexedDB for offline capability + - **Type-safe data handling**: Full TypeScript support with proper class instantiation + +7. **Advanced Audio Engine & Realistic Instruments**: + - **Professional Soundfont Integration**: High-quality instrument samples from FluidR3_GM soundfonts via remote CDN loading + - **Realistic Instrument Library**: Authentic piano, guitar, bass, and drum sounds replacing synthetic oscillators + - **Modular Audio Bus System**: Each track operates through a dedicated KGAudioBus with complete audio processing chain + - **Intelligent Buffer Management**: KGToneBuffersPool efficiently loads and caches audio samples (A0-C8 range) on-demand + - **Async Audio Factory Pattern**: KGToneSamplerFactory creates fully-loaded samplers with proper error handling + - **Real-time Audio Playback**: Full project playback with BPM-accurate timing using realistic instrument samples + - **Automatic Audio Context Management**: Browser autoplay policy compliance with graceful fallback + - **Advanced Track Controls**: Individual track volume, mute, solo with sophisticated audio routing + - **Note Preview with Real Instruments**: Immediate audio feedback using actual instrument samples with proper durations + - **Future-Ready Architecture**: Prepared for audio effects, filters, and advanced routing capabilities + +8. **Playhead & Transport Controls**: + - **Visual playhead indicator**: Blue-green vertical line with triangular marker showing current position + - **Interactive timeline navigation**: Click on bar numbers to jump playhead to nearest bar start + - **Dual-context rendering**: Playhead appears in both main track view and piano roll with proper positioning + - **Play/pause functionality**: Transport controls with state management and visual feedback + - **Timer-based playback**: BPM-accurate playhead movement during playback + - **Back to beginning**: Quick reset button to return playhead to project start + - **Automatic reset**: Playhead resets to position 0 when loading projects + - **Reactive state management**: Real-time UI updates synchronized between core engine and interface + +9. **Comprehensive Undo/Redo System**: + - **Command Pattern Architecture**: All operations implemented as undoable commands + - **Full Operation Coverage**: Undo/redo support for tracks, regions, notes, and project properties + - **Selective Property Tracking**: Only modified properties are tracked for efficient undo operations + - **Keyboard Shortcuts**: Ctrl/Cmd+Z for undo, Ctrl/Cmd+Y for redo with visual feedback + - **UI Integration**: Real-time undo/redo state displayed in toolbar with operation descriptions + - **Memory Management**: Configurable command history limits to prevent memory leaks + - **State Consistency**: Proper UI synchronization after undo/redo operations + - **Multi-Selection Support**: Batch operations on multiple selected items with single undo entry + +10. **Copy & Paste System**: + - **Keyboard shortcuts**: Ctrl/Cmd+C and Ctrl/Cmd+V for copy/paste operations + - **Toolbar buttons**: Dedicated Copy and Paste buttons in the main toolbar + - **Context-aware pasting**: Intelligent paste behavior based on current context (regions vs. notes) + - **Region copy/paste**: Copy and paste MIDI regions between tracks at playhead position with undo support + - **Note copy/paste**: Copy and paste MIDI notes within piano roll editor with undo support + - **Reusable utilities**: Centralized copy/paste logic in `copyPasteUtil.ts` for consistent behavior + - **Status feedback**: Real-time status messages showing copy/paste operation results + +11. **UI/UX**: + - Responsive layout + - Status bar with system messages + - Transport display (position, BPM, time signature) + - Component-based architecture for better maintainability + - Interactive cursors for different operations (grab/grabbing for moving, resize cursors for edges) + - Reusable UI components for consistent behavior (KGDropdown) + - Proper z-index management for overlapping UI elements + - Custom hooks for complex UI logic + +12. **Settings Panel**: + - General, Behavior, Templates sections with persistent config and debounced saves + - LLM provider switching without reload + +13. **AI Assistant**: + - Chat with slash-commands (`/clear`, `/welcome`) + - Region-aware messaging; tool-execution pipeline for music edits + +14. **Instrument Library**: + - General MIDI-based instrument browsing with icons and real soundfonts + - Instant note preview using the selected instrument + +15. **Project Upgrader**: + - Automatic migration of legacy projects to the latest structure version + +16. **Global Loading Overlay**: + - Shows while soundfonts download; safety timeout with user guidance if loading stalls + +## Features To Be Implemented + +1. **Audio Engine Enhancement**: + - Audio recording capabilities + - MIDI input/output support + - Real-time audio processing and effects + - Audio effects and plugin support + +2. **Advanced Playback Features**: + - Loop regions and loop playback + - Tempo automation and tempo changes during playback + - Audio-visual waveform synchronization + - Latency compensation and high-precision timing + +3. **Advanced Region Features**: + - Region splitting and merging + - Audio waveform visualization for audio regions + +4. **Advanced Track Features**: + - Track types (MIDI, audio, instrument, etc.) + - Track effects and processing + - Track automation + +5. **MIDI Editing**: + - Velocity editing + - MIDI CC automation + +6. **Audio Editing**: + - Waveform visualization + - Audio clip editing + - Audio effects + +7. **Advanced Project Management**: + - Export to audio formats + - Project templates + +8. **Performance Optimizations**: + - Virtualized track rendering + - Audio buffer management + - Worker threads for processing + +9. **Plugin System**: + - Virtual instrument support + - Effect plugin support + - Plugin browser and management + +## Development Guidelines + +1. **Architecture**: + - Maintain separation between core logic and UI + - Use TypeScript interfaces for all models + - Follow the singleton pattern for global services + - Implement Command Pattern for all undoable operations + - Structure UI as modular, self-contained components + - Follow component hierarchy with clear responsibilities + - Extract complex logic to custom hooks + +2. **State Management**: + - Keep UI state in Zustand stores + - Sync UI state with core models + - Use command pattern for all data modifications + - Execute commands through KGCore for undo/redo support + - Keep component state local when appropriate + - Use refs for tracking async state updates + +3. **UI Development**: + - Follow existing CSS patterns and variables + - Maintain responsive design + - Optimize for performance with React best practices + - Extract reusable components to improve maintainability + - Keep related functionality together in the same component + - Ensure proper z-index management for overlapping UI elements + - Organize components by functionality (piano, track, etc.) + +4. **Component Communication**: + - Parent components provide data via props + - Child components notify parents via callbacks + - UI components handle their own interactions + - Parent components coordinate data model updates + - Use custom hooks to share logic between components + +5. **Configuration and Constants**: + - Use centralized constants in the constants folder (`coreConstants.ts` for core, `uiConstants.ts` for UI) + - Default application settings defined in `/public/config.json` + - ConfigManager handles user customizations with persistent storage + - KGStorage provides generic access to IndexedDB for all storage needs + - Debug mode flags control console logging for different components + - Use refs for tracking async state updates + - Verify model updates after they occur + +6. **Testing**: + - Write unit tests for core logic + - Write integration tests for UI components + - Test audio processing with specialized audio testing tools + +## Current Limitations + +1. Limited keyboard shortcuts +2. Limited note editing capabilities (no velocity editing) +3. Export functionality UI only (no actual export implementation) +4. No audio recording capabilities yet +5. Limited audio effects and processing +6. No project browser UI (projects load/save by typing names) + +This document serves as a high-level overview of the KGStudio project, its architecture, current implementation, and future development plans. It should be updated as the project evolves to reflect the current state and goals. From c1112f41d5b6da94336e48082b376820b59c90c0 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Wed, 20 Aug 2025 15:48:26 -0700 Subject: [PATCH 5/7] added an option to fix the "Zoom can't capture the sound" problem (it is IMPORTANT to choose "share sound" when share screen in Zoom). --- public/config.json | 3 ++ .../settings/sections/BehaviorSettings.tsx | 31 +++++++++++++++ src/core/audio-interface/KGAudioInterface.ts | 38 +++++++++++++++++++ src/core/config/ConfigManager.ts | 6 +++ 4 files changed, 78 insertions(+) diff --git a/public/config.json b/public/config.json index 3072057..acdc552 100644 --- a/public/config.json +++ b/public/config.json @@ -54,6 +54,9 @@ "chatbox": { "default_open": true }, + "audio": { + "enable_audio_capture_for_screen_sharing": false + }, "templates": { "custom_instructions": "" } diff --git a/src/components/settings/sections/BehaviorSettings.tsx b/src/components/settings/sections/BehaviorSettings.tsx index 555680c..82fa4c5 100644 --- a/src/components/settings/sections/BehaviorSettings.tsx +++ b/src/components/settings/sections/BehaviorSettings.tsx @@ -3,6 +3,7 @@ import { ConfigManager } from '../../../core/config/ConfigManager'; const BehaviorSettings: React.FC = () => { const [chatboxDefaultOpen, setChatboxDefaultOpen] = useState(true); + const [enableAudioCapture, setEnableAudioCapture] = useState(false); const configManager = ConfigManager.instance(); @@ -14,6 +15,7 @@ const BehaviorSettings: React.FC = () => { } setChatboxDefaultOpen((configManager.get('chatbox.default_open') as boolean) ?? true); + setEnableAudioCapture((configManager.get('audio.enable_audio_capture_for_screen_sharing') as boolean) ?? false); }; loadConfig(); @@ -26,6 +28,12 @@ const BehaviorSettings: React.FC = () => { await configManager.set('chatbox.default_open', boolValue); }; + const handleEnableAudioCaptureChange = async (value: string) => { + const boolValue = value === 'yes'; + setEnableAudioCapture(boolValue); + await configManager.set('audio.enable_audio_capture_for_screen_sharing', boolValue); + }; + return (
@@ -50,6 +58,29 @@ const BehaviorSettings: React.FC = () => {
+ +
+

Audio

+ +
+ + +
+ Restart KGStudio (refresh the page) to take effect. Enable this option when KGStudio's audio cannot be captured during screen sharing in video calls (e.g., Zoom, Teams). This creates an additional audio stream that screen capture applications can detect. +
+ It is important to make sure when screen sharing in Zoom, the "Share Sound" option is enabled. +
+
+
); diff --git a/src/core/audio-interface/KGAudioInterface.ts b/src/core/audio-interface/KGAudioInterface.ts index 79850c1..a480145 100644 --- a/src/core/audio-interface/KGAudioInterface.ts +++ b/src/core/audio-interface/KGAudioInterface.ts @@ -7,6 +7,7 @@ import * as Tone from 'tone'; import { KGAudioBus } from './KGAudioBus'; import type { InstrumentType } from '../track/KGMidiTrack'; import { KGCore } from '../KGCore'; +import { ConfigManager } from '../config/ConfigManager'; /** * KGAudioInterface - Audio engine interface for the DAW @@ -32,6 +33,10 @@ export class KGAudioInterface { // Master volume control private masterGain: Tone.Gain | null = null; + // Audio capture for screen sharing + private captureDestination: MediaStreamAudioDestinationNode | null = null; + private captureStream: MediaStream | null = null; + // Private constructor to prevent direct instantiation private constructor() { console.log("KGAudioInterface initialized"); @@ -66,6 +71,14 @@ export class KGAudioInterface { Tone.Transport.bpm.value = TIME_CONSTANTS.DEFAULT_BPM; // Default BPM Tone.Transport.timeSignature = [TIME_CONSTANTS.DEFAULT_TIME_SIGNATURE.numerator, TIME_CONSTANTS.DEFAULT_TIME_SIGNATURE.denominator]; // Default time signature + // Check config and setup audio capture if enabled + const configManager = ConfigManager.instance(); + const enableCapture = configManager.get('audio.enable_audio_capture_for_screen_sharing') as boolean; + + if (enableCapture) { + this.setupAudioCapture(); + } + this.isInitialized = true; console.log("Audio engine initialized successfully"); } catch (error) { @@ -115,6 +128,12 @@ export class KGAudioInterface { this.masterGain = null; } + // Clean up capture resources + if (this.captureDestination) { + this.captureDestination = null; + this.captureStream = null; + } + this.isInitialized = false; this.isAudioContextStarted = false; @@ -602,8 +621,27 @@ export class KGAudioInterface { return Object.keys(FLUIDR3_INSTRUMENT_MAP) as InstrumentType[]; } + public getCaptureStream(): MediaStream | null { + return this.captureStream; + } + // ===== PRIVATE UTILITY METHODS ===== + /** + * Setup audio capture for screen sharing + */ + private setupAudioCapture(): void { + if (this.masterGain && !this.captureDestination) { + this.captureDestination = Tone.getContext().createMediaStreamDestination(); + this.captureStream = this.captureDestination.stream; + + // Connect master gain to both speakers AND capture destination + this.masterGain.connect(this.captureDestination); + + console.log('Audio capture enabled for screen sharing'); + } + } + /** * Check if any tracks are currently soloed */ diff --git a/src/core/config/ConfigManager.ts b/src/core/config/ConfigManager.ts index e540827..cafde76 100644 --- a/src/core/config/ConfigManager.ts +++ b/src/core/config/ConfigManager.ts @@ -60,6 +60,9 @@ interface AppConfig { chatbox: { default_open: boolean; }; + audio: { + enable_audio_capture_for_screen_sharing: boolean; + }; templates: { custom_instructions: string; }; @@ -205,6 +208,9 @@ export class ConfigManager { chatbox: { default_open: true }, + audio: { + enable_audio_capture_for_screen_sharing: false + }, templates: { custom_instructions: '' } From 0c5433240161658ce816d641bad9bef60ce2587e Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Thu, 21 Aug 2025 21:56:22 -0700 Subject: [PATCH 6/7] added export conversation feature. --- public/chat/export_conversation_template.md | 7 ++ src/App.css | 29 ++++++- src/components/ChatBox.tsx | 89 ++++++++++++++++++++- src/util/miscUtil.ts | 25 +++++- src/util/timeUtil.ts | 20 ++++- src/util/xmlUtil.ts | 19 ++++- 6 files changed, 181 insertions(+), 8 deletions(-) create mode 100644 public/chat/export_conversation_template.md diff --git a/public/chat/export_conversation_template.md b/public/chat/export_conversation_template.md new file mode 100644 index 0000000..00c2983 --- /dev/null +++ b/public/chat/export_conversation_template.md @@ -0,0 +1,7 @@ +# From: {role} + +{timestamp} + +{content} + +--- diff --git a/src/App.css b/src/App.css index c64ba48..8ce0d16 100644 --- a/src/App.css +++ b/src/App.css @@ -669,9 +669,9 @@ body { color: #e0e0e0; } -.export-dropdown .quant-dropdown { - width: 200px; - left: 0; +.chatbox-export-dropdown .quant-dropdown { + width: 250px; + left: -200px; } .key-signature-dropdown .quant-dropdown { @@ -1039,6 +1039,10 @@ body { flex-shrink: 0; } +.chatbox.is-hidden { + display: none; +} + /* Instrument Selection Panel */ .instrument-selection { display: flex; @@ -1197,6 +1201,25 @@ body { border-radius: 3px; } +/* ChatBox export button wrapper and dropdown positioning */ +.chatbox-export-wrapper { + position: relative; + display: inline-block; +} + +.chatbox-export-btn { + display: flex; + align-items: center; + gap: 4px; +} + +.chatbox-export-dropdown-anchor { + position: absolute; + top: 100%; + left: 0; + z-index: 10000; +} + .chatbox-header h3 { color: #e0e0e0; font-size: 12px; diff --git a/src/components/ChatBox.tsx b/src/components/ChatBox.tsx index 20b4e2b..ef77ce4 100644 --- a/src/components/ChatBox.tsx +++ b/src/components/ChatBox.tsx @@ -1,5 +1,5 @@ import React, { useState, useRef, useEffect, memo, useCallback } from 'react'; -import { FaPlus, FaBan } from 'react-icons/fa'; +import { FaPlus, FaBan, FaDownload } from 'react-icons/fa'; import { UserMessage, AssistantMessage } from './chat'; import { AgentCore } from '../agent/core/AgentCore'; import { OpenAIProvider } from '../agent/llm/OpenAIProvider'; @@ -14,6 +14,10 @@ import { processUserMessage } from '../util/messageFilter/UserMessageFilter'; import { useStreamProcessor } from '../hooks/useStreamProcessor'; import { createMessage, addWelcomeMessage } from '../utils/chatMessageUtils'; import { extractActionableTools, executeAllTools } from '../utils/toolExecutionUtils'; +import { formatLocalDateTime } from '../util/timeUtil'; +import { downloadBlob, buildTimestampSuffix } from '../util/miscUtil'; +import { wrapXmlBlocksInContent } from '../util/xmlUtil'; +import KGDropdown from './common/KGDropdown'; import type { ChatMessage } from '../types/projectTypes'; @@ -60,6 +64,65 @@ const ChatBox: React.FC = ({ isVisible }) => { // Track if this is the first message (for system prompt logging) const [isFirstMessage, setIsFirstMessage] = useState(true); + // Export dropdown state and options + const [showExportDropdown, setShowExportDropdown] = useState(false); + const exportOptions = [ + 'Export conversation as JSON', + 'Export conversation as Markdown' + ]; + + const handleExportOptionSelect = (option: string) => { + if (option === 'Export conversation as JSON') { + try { + const messages = AgentCore.instance().getAgentState().getMessages(); + const exportMessages = messages.map((m) => ({ + id: m.id, + role: m.role, + content: m.content, + timestamp: formatLocalDateTime(new Date(m.timestamp)) + })); + const json = JSON.stringify(exportMessages, null, 2); + const filename = `kgstudio-conversation-${buildTimestampSuffix()}.json`; + downloadBlob(json, 'application/json', filename); + } catch (err) { + console.error('Failed to export conversation as JSON:', err); + } + } else if (option === 'Export conversation as Markdown') { + (async () => { + try { + const messages = AgentCore.instance().getAgentState().getMessages(); + const templateUrl = `${import.meta.env.BASE_URL}chat/export_conversation_template.md`; + const res = await fetch(templateUrl); + const template = await res.text(); + + const isAutomatedUserMessage = (content: string): boolean => { + return /^tool:\s.*\nsuccess:\s*(true|false)/i.test(content); + }; + + const sections = messages.map((m) => { + const isAutomaticUserMessage = isAutomatedUserMessage(m.content); + const roleLabel = m.role === 'assistant' ? 'Assistant' : (isAutomaticUserMessage ? 'User (Automatic)' : 'User'); + const ts = formatLocalDateTime(new Date(m.timestamp)); + const contentWithXml = isAutomaticUserMessage ? "```\n" + m.content + "\n```" : wrapXmlBlocksInContent(m.content); + return template + .replace('{role}', roleLabel) + .replace('{timestamp}', ts) + .replace('{content}', contentWithXml); + }); + + const markdown = sections.join('\n'); + const filename = `kgstudio-conversation-${buildTimestampSuffix()}.md`; + downloadBlob(markdown, 'text/markdown', filename); + } catch (err) { + console.error('Failed to export conversation as Markdown:', err); + } + })(); + } else { + console.log('Chat export selected:', option); + } + setShowExportDropdown(false); + }; + // Message update callbacks for stream processor const handleMessageUpdate = useCallback((messageId: string, updater: (msg: ChatMessage) => ChatMessage) => { setMessages(prev => prev.map(msg => msg.id === messageId ? updater(msg) : msg)); @@ -323,7 +386,7 @@ const ChatBox: React.FC = ({ isVisible }) => { }, [isProcessing, isExecutingTools]); return ( -
+

K.G.Studio Musician Assistant

@@ -337,6 +400,28 @@ const ChatBox: React.FC = ({ isVisible }) => { )} +
+ +
+ +
+