From 2cd976ff9639d511a222244648356543f450985e Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Sun, 5 Apr 2026 18:59:25 -0700 Subject: [PATCH 1/6] refactor: migrate agent system from XML tool calling to OpenAI SDK with native function calling - Replace custom XML-based tool parsing (XMLToolExecutor) with OpenAI SDK's native tool_calls via `openai` npm package (dangerouslyAllowBrowser) - Consolidate 4 LLM providers (OpenAI, Claude, Gemini, ClaudeOpenRouter) into a single OpenAI SDK-based LLMProvider compatible with any OpenAI-style API (OpenAI, OpenRouter, Ollama, vLLM) - Move agentic tool execution loop from ChatBox into AgentCore - Update Message type to support tool roles, tool_calls, and tool_call_id - Update system prompt to remove XML formatting instructions (~45% smaller) - Remove AttemptCompletionTool (replaced by stop_reason detection), ThinkTool, and ThinkingTool - Polish tool descriptions for OpenAI function calling schema compliance - Normalize base URLs by stripping /chat/completions suffix --- README.md | 4 +- package-lock.json | 28 +- package.json | 1 + .../error_no_openai_compatible_base_url.md | 2 +- public/chat/help.md | 2 +- public/chat/welcome_new.md | 2 +- public/config.json | 2 +- public/prompts/system.md | 266 ++---------- src/agent/core/AgentCore.ts | 207 ++++++---- src/agent/core/AgentState.ts | 59 ++- src/agent/core/XMLToolExecutor.ts | 389 ------------------ src/agent/llm/ClaudeOpenRouterProvider.ts | 239 ----------- src/agent/llm/ClaudeProvider.ts | 162 -------- src/agent/llm/GeminiProvider.ts | 183 -------- src/agent/llm/LLMProvider.ts | 153 ++++++- src/agent/llm/OpenAIProvider.ts | 237 ----------- src/agent/llm/StreamingTypes.ts | 24 +- src/agent/tools/AddNotesTool.ts | 18 +- src/agent/tools/AttemptCompletionTool.ts | 49 --- src/agent/tools/BaseTool.ts | 88 +++- src/agent/tools/ReadMusicTool.ts | 10 +- src/agent/tools/RemoveNotesTool.ts | 10 +- src/agent/tools/ThinkTool.ts | 35 -- src/agent/tools/ThinkingTool.ts | 34 -- src/agent/tools/index.ts | 12 +- src/components/ChatBox.tsx | 236 +++-------- src/components/chat/AssistantMessage.tsx | 142 +------ .../settings/sections/GeneralSettings.tsx | 4 +- src/core/KGDebugger.ts | 192 +++------ src/core/config/ConfigManager.ts | 2 +- src/hooks/useStreamProcessor.ts | 113 +++-- src/utils/toolExecutionUtils.ts | 102 ----- 32 files changed, 697 insertions(+), 2310 deletions(-) delete mode 100644 src/agent/core/XMLToolExecutor.ts delete mode 100644 src/agent/llm/ClaudeOpenRouterProvider.ts delete mode 100644 src/agent/llm/ClaudeProvider.ts delete mode 100644 src/agent/llm/GeminiProvider.ts delete mode 100644 src/agent/llm/OpenAIProvider.ts delete mode 100644 src/agent/tools/AttemptCompletionTool.ts delete mode 100644 src/agent/tools/ThinkTool.ts delete mode 100644 src/agent/tools/ThinkingTool.ts delete mode 100644 src/utils/toolExecutionUtils.ts diff --git a/README.md b/README.md index 1fb737b..5f1f1fd 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ This project investigates how AI-human collaboration can enhance creative music- - In **Settings ⚙️ → General → LLM Provider**, select **OpenAI Compatible**. - In **OpenAI Compatible Server → Key**, paste your key. (Note: on non‑localhost, your key isn't persisted by default for security; you can enable "Persist API Keys on Non-Localhost" in Settings to persist them, though this may increase XSS risk.) - In **OpenAI Compatible Server → Model**, enter `qwen/qwen3-30b-a3b:free` or `qwen/qwen3-235b-a22b:free`. (Note: these are free models [qwen3-30b-a3b](https://openrouter.ai/qwen/qwen3-30b-a3b:free) and [qwen3-235b-a22b](https://openrouter.ai/qwen/qwen3-235b-a22b:free); non‑free models may require billing; some model providers may retain your data, check their privacy policies; this project is not affiliated with OpenRouter or any model provider.) - - In **OpenAI Compatible Server → Base URL**, enter `https://openrouter.ai/api/v1/chat/completions`. + - In **OpenAI Compatible Server → Base URL**, enter `https://openrouter.ai/api/v1`. (Alternatively, you can use the official OpenAI API, other OpenAI‑compatible services, or your own hosted LLM server. e.g., Ollama, vLLM) @@ -219,7 +219,7 @@ OpenRouter is a platform that provides unified access to a wide range of languag - `Anthropic: claude-4-sonnet` (`anthropic/claude-sonnet-4`: [Link](https://openrouter.ai/anthropic/claude-sonnet-4)) - `Qwen: qwen3-30b-a3b` (FREE MODEL: `qwen/qwen3-30b-a3b:free`: [Link](https://openrouter.ai/qwen/qwen3-30b-a3b:free)) - `Qwen: qwen3-235b-a22b` (FREE MODEL: `qwen/qwen3-235b-a22b:free`: [Link](https://openrouter.ai/qwen/qwen3-235b-a22b:free)) -6. Input the base URL `https://openrouter.ai/api/v1/chat/completions` **OpenAI Compatible Server → Base URL**. +6. Input the base URL `https://openrouter.ai/api/v1` **OpenAI Compatible Server → Base URL**. ### About the agent and LLM providers diff --git a/package-lock.json b/package-lock.json index be9f57e..76bc26e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,16 @@ { "name": "K.G.Studio", - "version": "0.5.4-build.20250830", + "version": "0.8.0-build.20260123", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "K.G.Studio", - "version": "0.5.4-build.20250830", + "version": "0.8.0-build.20260123", "dependencies": { "class-transformer": "^0.5.1", "idb": "^8.0.3", + "openai": "^6.33.0", "react": "^19.1.0", "react-dom": "^19.1.0", "react-icons": "^5.5.0", @@ -10879,6 +10880,27 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/openai": { + "version": "6.33.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-6.33.0.tgz", + "integrity": "sha512-xAYN1W3YsDXJWA5F277135YfkEk6H7D3D6vWwRhJ3OEkzRgcyK8z/P5P9Gyi/wB4N8kK9kM5ZjprfvyHagKmpw==", + "license": "Apache-2.0", + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "ws": "^8.18.0", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -14064,7 +14086,7 @@ "version": "8.18.3", "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=10.0.0" diff --git a/package.json b/package.json index c903a19..65e6035 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "dependencies": { "class-transformer": "^0.5.1", "idb": "^8.0.3", + "openai": "^6.33.0", "react": "^19.1.0", "react-dom": "^19.1.0", "react-icons": "^5.5.0", diff --git a/public/chat/error_no_openai_compatible_base_url.md b/public/chat/error_no_openai_compatible_base_url.md index 6f7a649..528de92 100644 --- a/public/chat/error_no_openai_compatible_base_url.md +++ b/public/chat/error_no_openai_compatible_base_url.md @@ -4,6 +4,6 @@ You selected OpenAI Compatible as your LLM Provider, but no Base URL is configur How to fix: - Go to **Settings ⚙️ → General → OpenAI Compatible Server** -- Enter the **Base URL** for your provider (e.g., `https://openrouter.ai/api/v1/chat/completions` for OpenRouter, `http://localhost:11434/api/chat` for Ollama, etc.) +- Enter the **Base URL** for your provider (e.g., `https://openrouter.ai/api/v1` for OpenRouter, `http://localhost:11434/api/chat` for Ollama, etc.) After updating your settings, try your request again. \ No newline at end of file diff --git a/public/chat/help.md b/public/chat/help.md index 6ce723c..d895e56 100644 --- a/public/chat/help.md +++ b/public/chat/help.md @@ -35,7 +35,7 @@ OpenRouter is a platform that provides unified access to a wide range of languag - `Anthropic: claude-4-sonnet` (`anthropic/claude-sonnet-4`: [Link](https://openrouter.ai/anthropic/claude-sonnet-4)) - `Qwen: qwen3-30b-a3b` (FREE MODEL: `qwen/qwen3-30b-a3b:free`: [Link](https://openrouter.ai/qwen/qwen3-30b-a3b:free)) - `Qwen: qwen3-235b-a22b` (FREE MODEL: `qwen/qwen3-235b-a22b:free`: [Link](https://openrouter.ai/qwen/qwen3-235b-a22b:free)) -6. Input the base URL `https://openrouter.ai/api/v1/chat/completions` of the OpenAI Compatible Server in **OpenAI Compatible Server → Base URL**. +6. Input the base URL `https://openrouter.ai/api/v1` of the OpenAI Compatible Server in **OpenAI Compatible Server → Base URL**. ### Basic DAW Operations diff --git a/public/chat/welcome_new.md b/public/chat/welcome_new.md index 124f2cf..3534e0d 100644 --- a/public/chat/welcome_new.md +++ b/public/chat/welcome_new.md @@ -7,7 +7,7 @@ Welcome to **K.G.Studio Musician Assistant** — your in‑browser AI partner fo - In **Settings ⚙️ → General → LLM Provider**, select **OpenAI Compatible**. - In **OpenAI Compatible Server → Key**, paste your key. Note: on non‑localhost, your key isn’t persisted; you’ll re‑enter it after refresh. Keep it safe. - In **OpenAI Compatible Server → Model**, enter `qwen/qwen3-30b-a3b:free`. Note: non‑free models may require billing. -- In **OpenAI Compatible Server → Base URL**, enter `https://openrouter.ai/api/v1/chat/completions`. +- In **OpenAI Compatible Server → Base URL**, enter `https://openrouter.ai/api/v1`. *(Alternatively, you can use the official OpenAI API, or any OpenAI‑compatible service)* diff --git a/public/config.json b/public/config.json index 60c1a47..fbfe08c 100644 --- a/public/config.json +++ b/public/config.json @@ -18,7 +18,7 @@ }, "claude_openrouter": { "api_key": "", - "base_url": "https://openrouter.ai/api/v1/chat/completions", + "base_url": "https://openrouter.ai/api/v1", "model": "anthropic/claude-sonnet-4.5" }, "openai_compatible": { diff --git a/public/prompts/system.md b/public/prompts/system.md index e5db53e..fa63775 100644 --- a/public/prompts/system.md +++ b/public/prompts/system.md @@ -14,177 +14,28 @@ You should leverage this extensive musical training to provide creative, musical TOOL USE -You have access to a set of tools that are executed upon the user's approval. You can use one tool per message, and will receive the result of that tool use in the user's response. You use tools step-by-step to accomplish a given task, with each tool use informed by the result of the previous tool use. Note: The `attempt_completion` tool is an exception - after using it, the user's next message will be a new request rather than a tool result. - -# Tool Use Formatting - -Tool use is formatted using XML-style tags. The tool name is enclosed in opening and closing tags, and each parameter is similarly enclosed within its own set of tags. Here's the structure: - - -value1 -value2 -... - - -For example: - - - 0 - 8 - - -Always adhere to this format for the tool use to ensure proper parsing and execution. +You have access to tools for reading and editing music. Tools are invoked via native function calling — you call them by name with structured parameters, and their results are returned to you automatically. Use tools step-by-step to accomplish a given task, with each tool call informed by the result of the previous one. When you have finished the task, respond with a final text message summarizing what you did. # Tools ## read_music -Description: Read a given part of the music. The output is the selected part of the music in ABC notation. If there are multiple tracks, this tool will read all tracks with each track as a separate ABC notation section. -Parameters: -- start_beat: (required) The start beat of the region to read. -- length: (optional) The length of the region to read. If you want to read the entire music, you can omit this parameter. -Usage: - - start from beat - length of the region to read (optional) - +Read existing musical content from the project. The output is in ABC notation. If there are multiple tracks, all tracks are returned as separate ABC notation sections, with track names (e.g., "Melody", "Bass", "Chords") providing arrangement context. ## remove_notes -Description: Remove notes from the given range in the current region. -Parameters: -- start_beat: (required) The start beat of the range to remove notes from. -- end_beat: (required) The end beat of the range to remove notes from. -Usage: - - start from beat - end at beat - +Remove notes from a given beat range in the current region. ## add_notes -Description: Add notes to the current region. -Parameters: -- notes: (required) The notes to add. Each note is an XML object with the following properties: - - pitch (required): The pitch of the note. - - start_beat (required): The start beat of the note. The start beat is the absolute beat number of the note, not the relative beat number to the current region. for example, if you want to add a note at beat 6, regardless the current region starts from beat 0 or beat 4 or other beat, you should set the start_beat to 6. - - length (required): The length of the note. -Usage: - - - - pitch of the first note you want to add, e.g. C4 - start beat of the first note you want to add - length of the first note you want to add - - - pitch of the second note you want to add, e.g. E4 - start beat of the second note you want to add - length of the second note you want to add - - ... - - +Add notes to the current region. Pitches use scientific pitch notation with support for sharps and flats (e.g., `C4`, `F#3`, `Bb2`). **Important**: the `start_beat` parameter is always the **absolute** beat position in the project timeline — not relative to the current region's start. For example, to place a note at beat 6, set `start_beat` to 6 regardless of where the current region begins. -## attempt_completion -Description: After each tool use, the user will respond with the result of that tool use, i.e. if it succeeded or failed, along with any reasons for failure. Once you've received the results of tool uses and can confirm that the task is complete, use this tool to present the result of your work to the user. -IMPORTANT NOTE: This tool CANNOT be used until you've confirmed from the user that any previous tool uses were successful. Failure to do so will result in code corruption and system failure. Before using this tool, you must ask yourself in tags if you've confirmed from the user that any previous tool uses were successful. If not, then DO NOT use this tool. -Parameters: -- comment: (required) The result of the task. Formulate this result in a way that is final and does not require further input from the user. Don't end your result with questions or offers for further assistance. -Usage: - - Your final result or comment here - - -# Tool Use Examples - -## Example 1: Requesting to read a part of the music from beat 0 to beat 8 - - - 0 - 8 - - -## Example 2: Requesting to remove notes from the current region from beat 0 to beat 4 - - - 0 - 4 - - -## Example 3: Requesting to add notes C4, D4, G4, E4 to the current region, starting at beat 0 and each one lasting 1 beat. - - - - - C4 - 0 - 1 - - - D4 - 1 - 1 - - - G4 - 2 - 1 - - - E4 - 3 - 1 - - - - -## Example 4: Requesting to add a chord containing C4, E4, G4 to the current region, starting at beat 0 and lasting 2 beats. - - - - - C4 - 0 - 2 - - - E4 - 0 - 2 - - - G4 - 0 - 2 - - - - - -## Example 5: Requesting to complete current task with a final comment - - - Completed the I-V-vi-IV chord progression in C major, each chord lasting 2 beats - +To create a melodic line, use sequential `start_beat` values for each note. To create a chord, give multiple notes the same `start_beat`. # Tool Use Guidelines -1. In tags, assess what information you already have and what information you need to proceed with the task. -2. Choose the most appropriate tool based on the task and the tool descriptions provided. Assess if you need additional information to proceed, and which of the available tools would be most effective for gathering this information. It's critical that you think about each available tool and use the one that best fits the current step in the task. -3. If multiple actions are needed, use one tool at a time per message to accomplish the task iteratively, with each tool use being informed by the result of the previous tool use. Do not assume the outcome of any tool use. Each step must be informed by the previous step's result. -4. Formulate your tool use using the XML format specified for each tool. -5. After each tool use, the user will respond with the result of that tool use. This result will provide you with the necessary information to continue your task or make further decisions. This response may include: - - Information about whether the tool succeeded or failed, along with any reasons for failure. - - Music pieces in ABC notation if you have used the read_music tool. - - Any other relevant feedback or information related to the tool use. - Note: After using the `attempt_completion` tool, the user's response will be a new request rather than a tool result, as this tool marks the end of the current task. -6. ALWAYS wait for user confirmation after each tool use before proceeding. Never assume the success of a tool use without explicit confirmation of the result from the user. Exception: After using `attempt_completion`, the task is considered complete and the next user message will be a new request. - -It is crucial to proceed step-by-step, waiting for the user's message after each tool use before moving forward with the task. This approach allows you to: -1. Confirm the success of each step before proceeding. -2. Address any issues or errors that arise immediately. -3. Adapt your approach based on new information or unexpected results. -4. Ensure that each action builds correctly on the previous ones. - -By waiting for and carefully considering the user's response after each tool use, you can react accordingly and make informed decisions about how to proceed with the task. This iterative process helps ensure the overall success and accuracy of your work. +1. Assess what information you already have and what you need before choosing a tool. +2. Choose the most appropriate tool for the current step. If you need to understand existing music, use `read_music` first. +3. After each tool call, examine the result before deciding the next action. Do not assume success — verify from the returned result. +4. If a required parameter cannot be determined from context, ask the user instead of guessing. +5. Proceed step-by-step. Each action should build on confirmed results from previous steps. ==== @@ -205,7 +56,7 @@ You have access to two tools for working with the current music region: **remove ## Important Considerations -- If you have used the add_notes tool previously to add notes to the current region, you should use this tool to remove the notes you added before using the add_notes tool again. +- If you have previously added notes to the current region, you should use this tool to remove those notes before using `add_notes` again. - Ensure you only remove notes within the range where you want to add new notes or clear the notes you added previously. # add_notes @@ -217,13 +68,12 @@ You have access to two tools for working with the current music region: **remove ## When to Use - Add notes to the current region. -- You should add notes one by one. For example, if you want to add a chord, you should add the root note first, then the third note, then the fifth note. -- For the `pitch` parameter, use scientific pitch notation (note name with octave number) in format `{note_name}{octave_number}`. For example, `C4` is the C note in the 4th octave. +- For the `pitch` parameter, use scientific pitch notation in format `{note_name}{accidental}{octave_number}`. For example, `C4` is middle C, `F#3` is F-sharp in the 3rd octave, and `Bb2` is B-flat in the 2nd octave. ## Important Considerations -- **Do not omit notes**: It is important that when adding notes, you must explicitly output every note that should be added — do not omit, summarize, or replace them with comments like “...”. Even if the pattern is repetitive, list all notes in full detail in the correct order. NEVER OMIT ANY NOTES IN THE XML BECAUSE OF REPETITION!! -- **Reading Music**: You should NEVER ask the user to manually provide you music pieces BEFORE invoking the read_music tool. Always use the read_music tool to get the music pieces first. +- **Do not omit notes**: When adding notes, you must explicitly include every note — do not omit, summarize, or replace them with comments like "...". Even if the pattern is repetitive, list all notes in full detail. NEVER OMIT ANY NOTES BECAUSE OF REPETITION. +- **Reading Music**: You should NEVER ask the user to manually provide you music pieces BEFORE invoking the `read_music` tool. Always use `read_music` to get the music pieces first. - **Music Validation**: Always validate your musical choices: - Ensure pitches are within reasonable ranges for the current instrument - Verify that note timings align with the current time signature @@ -231,79 +81,11 @@ You have access to two tools for working with the current music region: **remove - Confirm note lengths don't extend beyond reasonable musical phrases - **Pitch Notation**: Use scientific pitch notation (e.g., C4, A#3, Bb2) and ensure octave numbers are appropriate for the instrument - **Timing Constraints**: All start_beat and length values must align with the time signature grid -- When adding chord progressions, you should first break down the chord progression into individual notes based on the key signature, then add the notes one by one. -- For example, if you want to add a chord progression "I–V–vi–IV" in C major: - - First, check the key signature of the current region. If it's C major, then the chord progression should be "C–G–Am–F". - - Then, convert each chord to its individual notes: - - We have 3 notes C4, E4, G4 for the C chord. - - We have 3 notes G3, B3, D4 for the G chord. - - We have 3 notes A3, C4, E4 for the Am chord. - - We have 3 notes F3, A3, C4 for the F chord. - - Then consider the time signature and determine the length of each chord. If you determine the length of each chord is 4 beats, then the XML you should use to create the chord progression using the **add_notes** tool is: - - - - C4 - 0 - 4 - - - E4 - 0 - 4 - - - G4 - 0 - 4 - - - G3 - 4 - 4 - - - B3 - 4 - 4 - - - D4 - 4 - 4 - - - A3 - 8 - 4 - - - C4 - 8 - 4 - - - E4 - 8 - 4 - - - F3 - 12 - 4 - - - A3 - 12 - 4 - - - C4 - 12 - 4 - - - +- When adding chord progressions, first break down the progression into individual notes based on the key signature, then add all the notes. +- For example, to create a I-V-vi-IV progression in C major with 4-beat chords: + 1. Check the key signature. In C major, the chords are C, G, Am, F. + 2. Convert each chord to individual notes: C = C4/E4/G4, G = G3/B3/D4, Am = A3/C4/E4, F = F3/A3/C4. + 3. Call `add_notes` with all 12 notes: the C chord notes at `start_beat` 0, the G chord notes at `start_beat` 4, the Am chord notes at `start_beat` 8, and the F chord notes at `start_beat` 12 — each with `length` 4. # Workflow Tips @@ -326,11 +108,11 @@ OBJECTIVE You accomplish a given task iteratively, breaking it down into clear steps and working through them methodically. 1. Analyze the user's task and set clear, achievable goals to accomplish it. Prioritize these goals in a logical order. -2. Work through these goals sequentially, utilizing available tools one at a time as necessary. Each goal should correspond to a distinct step in your problem-solving process. -3. Remember, you have extensive capabilities with access to a wide range of tools that can be used in powerful and clever ways as necessary to accomplish each goal. Before calling a tool, do some analysis within tags. First, read the existing music to get the context. Then, think about which of the provided tools is the most relevant tool to accomplish the user's task. Next, go through each of the required parameters of the relevant tool and determine if the user has directly provided or given enough information to infer a value. When deciding if the parameter can be inferred, carefully consider all the context to see if it supports a specific value. If all of the required parameters are present or can be reasonably inferred, close the thinking tag and proceed with the tool use. BUT, if one of the values for a required parameter is missing, DO NOT invoke the tool (not even with fillers for the missing params) and instead, ask the user to provide the missing parameters without any tool invoking (which will automatically pause the task execution). DO NOT ask for more information on optional parameters if it is not provided. -4. Once you've completed the user's task, you must use the attempt_completion tool to present the result of the task to the user. +2. Work through these goals sequentially, utilizing available tools as necessary. Each goal should correspond to a distinct step in your problem-solving process. +3. Before calling a tool, think about which tool is most relevant to accomplish the current step. Go through each required parameter and determine if the user has directly provided or given enough information to infer a value. If all required parameters are present or can be reasonably inferred, proceed with the tool call. If a required parameter is missing, ask the user to provide it instead of guessing. +4. Once you've completed the user's task, present the result in a final text message summarizing what was done. 5. The user may provide feedback, which you can use to make improvements and try again. But DO NOT continue in pointless back and forth conversations, i.e. don't end your responses with questions or offers for further assistance. -6. It is important to think about the task step by step. DO NOT directly jump to tool invocation without thinking. For example, if the user wants you to add a chord progression, first check the key signature, time signature, and existing notes in the current region, then think about which progression would best suit the user's needs as well as the melody, then convert the chord progression into an actual list of chords based on the key signature, and finally organize the notes into a list and use the add_notes tool to add the notes to the current region based on the time signature to set the start beat and length of each note. +6. It is important to think about the task step by step. DO NOT directly jump to tool invocation without thinking. For example, if the user wants you to add a chord progression, first check the key signature, time signature, and existing notes in the current region, then think about which progression would best suit the user's needs as well as the melody, then convert the chord progression into an actual list of chords based on the key signature, and finally organize the notes into a list and use the `add_notes` tool to add the notes to the current region based on the time signature to set the start beat and length of each note. ==== @@ -433,7 +215,7 @@ When working with drum tracks, use these pitch mappings for accurate drum notati - **A5 (81)**: Open Triangle - Open triangle **Usage Notes for Drums**: -- When composing for drums, use the scientific pitch notation (e.g., C2, D2, F#2) in your add_notes commands +- When composing for drums, use the scientific pitch notation (e.g., C2, D2, F#2) in your `add_notes` tool calls - Focus on the core drum kit sounds (36, 38, 42, 46, 49, 51) for basic patterns - Use extended percussion for more complex arrangements and world music styles - Consider the musical context when selecting appropriate drum sounds diff --git a/src/agent/core/AgentCore.ts b/src/agent/core/AgentCore.ts index 66ea05a..c943c3d 100644 --- a/src/agent/core/AgentCore.ts +++ b/src/agent/core/AgentCore.ts @@ -1,113 +1,188 @@ import { LLMProvider } from '../llm/LLMProvider'; import { AgentState } from './AgentState'; import { SystemPrompts } from './SystemPrompts'; +import { AVAILABLE_TOOLS } from '../tools'; +import { useProjectStore } from '../../stores/projectStore'; import type { StreamChunk } from '../llm/StreamingTypes'; +import type { ToolCall } from './AgentState'; +import type { OpenAIToolDefinition } from '../tools/BaseTool'; /** - * Main orchestrator for the AI agent system + * Main orchestrator for the AI agent system. + * Handles the full agentic loop: LLM streaming → tool execution → result feedback → repeat. */ export class AgentCore { private static _instance: AgentCore | null = null; - + private llmProvider: LLMProvider | null = null; private agentState: AgentState; private currentUserMessageId: string | null = null; private currentAssistantMessageId: string | null = null; - + private constructor() { this.agentState = new AgentState(); } - - /** - * Get the singleton instance - */ + static instance(): AgentCore { if (!AgentCore._instance) { AgentCore._instance = new AgentCore(); } return AgentCore._instance; } - - /** - * Set the LLM provider - */ + setLLMProvider(provider: LLMProvider): void { this.llmProvider = provider; } - - /** - * Get the current LLM provider - */ + getLLMProvider(): LLMProvider | null { return this.llmProvider; } - - /** - * Get the agent state - */ + getAgentState(): AgentState { return this.agentState; } - + /** - * Process user input and generate streaming response + * Get OpenAI tool definitions for all available tools + */ + private getToolDefinitions(): OpenAIToolDefinition[] { + return Object.values(AVAILABLE_TOOLS).map(ToolClass => { + const tool = new ToolClass(); + return tool.getDefinition(); + }); + } + + /** + * Execute a single tool call and return the result + */ + private async executeTool(toolCall: ToolCall): Promise<{ success: boolean; result: string }> { + const toolName = toolCall.function.name; + const ToolClass = AVAILABLE_TOOLS[toolName as keyof typeof AVAILABLE_TOOLS]; + + if (!ToolClass) { + return { success: false, result: `Unknown tool: ${toolName}` }; + } + + try { + const params = JSON.parse(toolCall.function.arguments); + const toolInstance = new ToolClass(); + const result = await toolInstance.execute(params); + + // Sync UI state after successful tool execution + if (result.success) { + useProjectStore.getState().refreshProjectState(); + } + + return result; + } catch (error) { + return { success: false, result: `Tool execution failed: ${error}` }; + } + } + + /** + * Process user input and generate streaming response. + * Handles the full agentic loop internally: if the LLM returns tool_calls, + * execute them and feed results back until the LLM produces a final text response. */ async *processUserInput(userInput: string): AsyncIterableIterator { if (!this.llmProvider) { throw new Error('No LLM provider configured'); } - - // Add user message to state and track its ID + + // Add user message to state this.currentUserMessageId = this.agentState.addMessage('user', userInput); - - // Get system prompt with current context + const systemPrompt = await SystemPrompts.getSystemPromptWithContext(); - - // Get full conversation history with preserved roles - const conversationHistory = this.agentState.getMessages(); - - // Generate streaming response with full conversation context - let assistantResponse = ''; - - // Pre-add an empty assistant message that we'll update as we stream - this.currentAssistantMessageId = this.agentState.addMessage('assistant', ''); - + const tools = this.getToolDefinitions(); + try { - for await (const chunk of this.llmProvider.generateStream(conversationHistory, systemPrompt)) { - if (chunk.type === 'text') { - assistantResponse += chunk.content; - // Update the assistant message in real-time - this.agentState.updateMessage(this.currentAssistantMessageId, assistantResponse); + // Agentic loop: stream → check for tool calls → execute → repeat + let continueLoop = true; + + while (continueLoop) { + const conversationHistory = this.agentState.getMessages(); + + // Pre-add an empty assistant message that we'll update as we stream + this.currentAssistantMessageId = this.agentState.addMessage('assistant', ''); + + let assistantTextContent = ''; + const accumulatedToolCalls: ToolCall[] = []; + let finishReason = 'stop'; + + for await (const chunk of this.llmProvider.generateStream(conversationHistory, systemPrompt, tools)) { + if (chunk.type === 'text') { + assistantTextContent += chunk.content; + this.agentState.updateMessage(this.currentAssistantMessageId, assistantTextContent); + yield chunk; + } else if (chunk.type === 'tool_call' && chunk.toolCall) { + accumulatedToolCalls.push(chunk.toolCall); + } else if (chunk.type === 'done') { + finishReason = chunk.finishReason ?? 'stop'; + } + } + + if (finishReason === 'tool_calls' && accumulatedToolCalls.length > 0) { + // Update assistant message with tool calls + this.agentState.updateMessage( + this.currentAssistantMessageId, + assistantTextContent || null, + { tool_calls: accumulatedToolCalls } + ); + + // Execute each tool call and add results to conversation + for (const toolCall of accumulatedToolCalls) { + // Notify UI about the tool call + yield { type: 'tool_call', content: '', toolCall }; + + const result = await this.executeTool(toolCall); + + // Add tool result message to conversation history + this.agentState.addMessage('tool', JSON.stringify(result), { + tool_call_id: toolCall.id, + }); + + // Notify UI about the tool result + yield { + type: 'tool_result', + content: '', + toolResult: { + name: toolCall.function.name, + success: result.success, + result: result.result, + }, + }; + } + + // Clear assistant message ID before next iteration creates a new one + this.currentAssistantMessageId = null; + + // Continue loop — send tool results back to LLM + } else { + // LLM finished with text response (stop reason) + this.agentState.updateMessage(this.currentAssistantMessageId, assistantTextContent); + continueLoop = false; } - yield chunk; - } - - // Final update to ensure the complete response is stored - if (assistantResponse) { - this.agentState.updateMessage(this.currentAssistantMessageId, assistantResponse); } + + yield { type: 'done', content: '', finishReason: 'stop' }; } finally { - // Clear the current message IDs when done (successfully or not) this.currentUserMessageId = null; this.currentAssistantMessageId = null; } } - - + /** * Abort the current streaming request and clean up messages * Returns the content of the user message that was aborted (for restoring to input) */ abortCurrentRequest(): string | null { let userMessageContent = null; - - // Remove the current assistant message (the "in progress" one) + if (this.currentAssistantMessageId) { this.agentState.removeMessage(this.currentAssistantMessageId); this.currentAssistantMessageId = null; } - - // Remove the current user message and get its content for restoration + if (this.currentUserMessageId) { const messages = this.agentState.getMessages(); const userMessage = messages.find(msg => msg.id === this.currentUserMessageId); @@ -117,35 +192,23 @@ export class AgentCore { this.agentState.removeMessage(this.currentUserMessageId); this.currentUserMessageId = null; } - + return userMessageContent; } - - /** - * Check if there's a current streaming request in progress - */ + isStreamingInProgress(): boolean { - return this.currentUserMessageId !== null && this.currentAssistantMessageId !== null; + return this.currentUserMessageId !== null; } - - /** - * Clear the conversation history - */ + clearConversation(): void { this.agentState.clearMessages(); } - - /** - * Get whether the agent is currently working on a task - */ + getIsWorkingOnTask(): boolean { return this.agentState.getIsWorkingOnTask(); } - - /** - * Set whether the agent is currently working on a task - */ + setIsWorkingOnTask(isWorking: boolean): void { this.agentState.setIsWorkingOnTask(isWorking); } -} \ No newline at end of file +} diff --git a/src/agent/core/AgentState.ts b/src/agent/core/AgentState.ts index f7f405e..d860a78 100644 --- a/src/agent/core/AgentState.ts +++ b/src/agent/core/AgentState.ts @@ -2,50 +2,73 @@ * Manages the state of an agent conversation */ +/** + * Tool call info attached to assistant messages (OpenAI function calling format) + */ +export interface ToolCall { + id: string; + type: 'function'; + function: { + name: string; + arguments: string; // JSON string of parameters + }; +} + export interface Message { id: string; - role: 'user' | 'assistant'; - content: string; + role: 'user' | 'assistant' | 'tool'; + content: string | null; timestamp: number; + tool_calls?: ToolCall[]; // present on assistant messages when LLM invokes tools + tool_call_id?: string; // present on tool-result messages, links back to ToolCall.id } export class AgentState { private messages: Message[] = []; private conversationId: string; private isWorkingOnTask: boolean = false; - + constructor(conversationId?: string, isWorkingOnTask: boolean = false) { this.conversationId = conversationId || this.generateConversationId(); this.isWorkingOnTask = isWorkingOnTask; } - + /** * Add a message to the conversation */ - addMessage(role: 'user' | 'assistant', content: string): string { + addMessage( + role: 'user' | 'assistant' | 'tool', + content: string | null, + options?: { tool_calls?: ToolCall[]; tool_call_id?: string } + ): string { const message: Message = { id: this.generateMessageId(), role, content, - timestamp: Date.now() + timestamp: Date.now(), + ...(options?.tool_calls ? { tool_calls: options.tool_calls } : {}), + ...(options?.tool_call_id ? { tool_call_id: options.tool_call_id } : {}), }; - + this.messages.push(message); return message.id; } - + /** * Update the content of a message by ID */ - updateMessage(messageId: string, content: string): boolean { + updateMessage(messageId: string, content: string | null, options?: { tool_calls?: ToolCall[] }): boolean { const messageIndex = this.messages.findIndex(msg => msg.id === messageId); if (messageIndex !== -1) { this.messages[messageIndex].content = content; + if (options?.tool_calls) { + this.messages[messageIndex].tool_calls = options.tool_calls; + } return true; } return false; } - + /** * Remove a message by ID */ @@ -57,46 +80,46 @@ export class AgentState { } return false; } - + /** * Remove the last N messages */ removeLastMessages(count: number): void { this.messages.splice(-count, count); } - + /** * Get all messages in the conversation */ getMessages(): Message[] { return [...this.messages]; } - + /** * Get the conversation ID */ getConversationId(): string { return this.conversationId; } - + /** * Clear all messages */ clearMessages(): void { this.messages = []; } - + /** * Get the last N messages */ getRecentMessages(count: number): Message[] { return this.messages.slice(-count); } - + private generateConversationId(): string { return `conv_${Date.now()}_${Math.random().toString(36).substring(2, 15)}`; } - + private generateMessageId(): string { return `msg_${Date.now()}_${Math.random().toString(36).substring(2, 15)}`; } @@ -109,4 +132,4 @@ export class AgentState { setIsWorkingOnTask(isWorkingOnTask: boolean): void { this.isWorkingOnTask = isWorkingOnTask; } -} \ No newline at end of file +} diff --git a/src/agent/core/XMLToolExecutor.ts b/src/agent/core/XMLToolExecutor.ts deleted file mode 100644 index 6694ccf..0000000 --- a/src/agent/core/XMLToolExecutor.ts +++ /dev/null @@ -1,389 +0,0 @@ -/** - * XMLToolExecutor - Bridge between XML tool invocations and the existing tool system - * Parses XML blocks from LLM responses and executes corresponding tools - */ - -import { extractXMLFromString } from '../../util/xmlUtil'; -import { AVAILABLE_TOOLS, type ToolName } from '../tools'; -import type { BaseTool, ToolResult } from '../tools/BaseTool'; -import { useProjectStore } from '../../stores/projectStore'; - -/** - * Main executor class for XML-based tool invocations - * Integrates with existing tool architecture and streaming types - */ -export class XMLToolExecutor { - // Private static instance for singleton pattern - private static _instance: XMLToolExecutor | null = null; - - // Private constructor to prevent direct instantiation - private constructor() {} - - /** - * Get the singleton instance of XMLToolExecutor - */ - public static instance(): XMLToolExecutor { - if (!XMLToolExecutor._instance) { - XMLToolExecutor._instance = new XMLToolExecutor(); - } - return XMLToolExecutor._instance; - } - - /** - * Execute all XML tool invocations found in the given input string - * @param input - String containing XML tool invocations (typically LLM response) - * @returns Promise resolving to array of tool results in order of appearance - */ - public async executeXMLTools(input: string): Promise { - try { - // Extract all XML blocks from the input - const xmlBlocks = extractXMLFromString(input); - - if (xmlBlocks.length === 0) { - return []; - } - - // Process each XML block and collect results - const results: ToolResult[] = []; - - for (const xmlBlock of xmlBlocks) { - try { - const result = await this.executeXMLBlock(xmlBlock); - results.push(result); - } catch (error) { - // Create failed result - results.push({ - success: false, - result: `Failed to process XML block: ${error}` - }); - } - } - - return results; - - } catch (error) { - return [{ - success: false, - result: `Failed to execute XML tools: ${error}` - }]; - } - } - - /** - * Execute a single XML block as a tool invocation - * @param xmlBlock - XML string representing a tool invocation - * @returns Promise resolving to tool execution result - */ - private async executeXMLBlock(xmlBlock: string): Promise { - // Parse XML to extract tool information - const parseResult = this.parseXMLBlock(xmlBlock); - - if (!parseResult.success) { - return { - success: false, - result: parseResult.error || 'Failed to parse XML block' - }; - } - - // Check if tool exists in registry - if (!(parseResult.toolName in AVAILABLE_TOOLS)) { - return { - success: false, - result: `Unknown tool: ${parseResult.toolName}` - }; - } - - try { - // Create tool instance - const ToolClass = AVAILABLE_TOOLS[parseResult.toolName as ToolName]; - const toolInstance: BaseTool = new ToolClass(); - - // Execute the tool - const toolResult = await toolInstance.execute(parseResult.parameters); - - // Sync UI state if the tool execution was successful - if (toolResult.success) { - this.syncUIState(); - } - - return toolResult; - - } catch (error) { - return { - success: false, - result: `Tool execution failed: ${error}` - }; - } - } - - /** - * Parse XML block to extract tool name and parameters - * @param xmlBlock - XML string to parse - * @returns Parse result with tool information or error - */ - private parseXMLBlock(xmlBlock: string): { success: boolean; toolName: string; parameters: Record; error?: string } { - try { - // Special pre-processing for attempt_completion: ensure is wrapped in CDATA - const preparedXml = this.preprocessAttemptCompletionXML(xmlBlock); - - // Parse XML using native DOMParser - const parser = new DOMParser(); - const doc = parser.parseFromString(preparedXml, 'text/xml'); - - // Check for parsing errors - const parserError = doc.querySelector('parsererror'); - if (parserError) { - return { - success: false, - toolName: '', - parameters: {}, - error: `XML parsing error: ${parserError.textContent}` - }; - } - - // Get the root element (tool name) - const rootElement = doc.documentElement; - const toolName = rootElement.tagName; - - // Parse XML parameters - const parameters = this.parseXMLParameters(rootElement); - - return { - success: true, - toolName, - parameters - }; - - } catch (error) { - return { - success: false, - toolName: '', - parameters: {}, - error: `Failed to parse XML: ${error}` - }; - } - } - - /** - * Ensure special tools have CDATA-wrapped content where appropriate. - * - attempt_completion: wrap inner text with CDATA (if not already) - * - think / thinking: wrap root inner text with CDATA (if not already) - * Decode basic XML entities before wrapping so CDATA contains human-readable text. - */ - private preprocessAttemptCompletionXML(xml: string): string { - try { - const leadingWhitespaceMatch = xml.match(/^\s*/); - const prefix = leadingWhitespaceMatch ? leadingWhitespaceMatch[0] : ''; - const withoutLeading = xml.slice(prefix.length); - const rootMatch = withoutLeading.match(/^<([A-Za-z_][\w-]*)\b/); - const root = rootMatch?.[1] || ''; - if (root !== 'attempt_completion' && root !== 'think' && root !== 'thinking') return xml; - - // Helper to decode entities - const decodeEntities = (text: string): string => - text - .replace(/</g, '<') - .replace(/>/g, '>') - .replace(/&/g, '&') - .replace(/"/g, '"') - .replace(/'/g, "'"); - - if (root === 'attempt_completion') { - // Find first ... - const commentRegex = /([\s\S]*?)<\/comment>/i; - const match = xml.match(commentRegex); - if (!match) return xml; - - const inner = match[1]; - if (/`; - return xml.replace(commentRegex, replacement); - } - - // Handle ... or ... - const rootRegex = new RegExp(`<${root}>([\\s\\S]*?)`, 'i'); - const rootMatchContent = xml.match(rootRegex); - if (!rootMatchContent) return xml; - const innerRoot = rootMatchContent[1]; - if (/`; - return xml.replace(rootRegex, replacementRoot); - } catch { - // On any error, return original XML to avoid breaking flow - return xml; - } - } - - /** - * Parse XML element into tool parameters object - * Converts XML structure to JavaScript object that matches tool parameter schema - * @param element - Root XML element containing tool parameters - * @returns Parameters object for tool execution - */ - private parseXMLParameters(element: Element): Record { - const parameters: Record = {}; - - // Special handling for thinking tool: if no child elements, use text content directly - if (element.tagName === 'thinking' && element.children.length === 0) { - const textContent = element.textContent?.trim() || ''; - parameters.content = textContent; - return parameters; - } - - // Process all child elements - for (const child of element.children) { - const paramName = child.tagName; - const paramValue = this.parseXMLValue(child); - - // Handle arrays (multiple elements with same tag name) - if (parameters[paramName] !== undefined) { - // Convert to array if not already - if (!Array.isArray(parameters[paramName])) { - parameters[paramName] = [parameters[paramName]]; - } - (parameters[paramName] as unknown[]).push(paramValue); - } else { - parameters[paramName] = paramValue; - } - } - - // Apply array wrapper flattening - return this.flattenArrayWrappers(parameters); - } - - /** - * Flatten array wrapper patterns in parsed parameters - * Converts structures like {notes: {note: [...]}} to {notes: [...]} - * @param parameters - Parsed parameters object - * @returns Parameters with flattened array wrappers - */ - private flattenArrayWrappers(parameters: Record): Record { - const flattened: Record = {}; - - for (const [key, value] of Object.entries(parameters)) { - if (this.isArrayWrapperCandidate(key, value)) { - // This is an array wrapper - flatten it - const wrapperObj = value as Record; - const innerKeys = Object.keys(wrapperObj); - - if (innerKeys.length === 1) { - const innerKey = innerKeys[0]; - const innerValue = wrapperObj[innerKey]; - - // Check if inner key is singular form of outer key - if (this.isSingularOf(innerKey, key)) { - // Flatten: {notes: {note: [...]}} → {notes: [...]} - flattened[key] = innerValue; - continue; - } - } - } - - // No flattening needed, keep as is - flattened[key] = value; - } - - return flattened; - } - - /** - * Check if a value is a candidate for array wrapper flattening - * @param key - Parameter key (e.g., "notes") - * @param value - Parameter value to check - * @returns True if this looks like an array wrapper pattern - */ - private isArrayWrapperCandidate(key: string, value: unknown): boolean { - // Must be an object (not array, not primitive) - if (typeof value !== 'object' || value === null || Array.isArray(value)) { - return false; - } - - const obj = value as Record; - const innerKeys = Object.keys(obj); - - // Must have exactly one property - if (innerKeys.length !== 1) { - return false; - } - - const innerKey = innerKeys[0]; - const innerValue = obj[innerKey]; - - // Inner value should be an array or could become an array - // (single items are often converted to arrays by the parser) - return this.isSingularOf(innerKey, key) && - (Array.isArray(innerValue) || typeof innerValue === 'object'); - } - - /** - * Check if one word is the singular form of another (simple heuristic) - * @param singular - Potential singular form (e.g., "note") - * @param plural - Potential plural form (e.g., "notes") - * @returns True if singular appears to be singular form of plural - */ - private isSingularOf(singular: string, plural: string): boolean { - // Simple heuristics for common English pluralization - if (plural === singular + 's') return true; // note → notes - if (plural === singular + 'es') return true; // box → boxes - if (plural.endsWith('ies') && singular.endsWith('y')) { // entry → entries - return plural === singular.slice(0, -1) + 'ies'; - } - - // Add more rules as needed for your specific use cases - return false; - } - - /** - * Parse a single XML element value, handling different data types and structures - * @param element - XML element to parse - * @returns Parsed value (string, number, boolean, object, or array) - */ - private parseXMLValue(element: Element): unknown { - // If element has children, parse as object - if (element.children.length > 0) { - return this.parseXMLParameters(element); - } - - // Get text content - const textContent = element.textContent?.trim() || ''; - - // Try to parse as number - if (/^-?\d+(\.\d+)?$/.test(textContent)) { - return parseFloat(textContent); - } - - // Try to parse as boolean - if (textContent === 'true') return true; - if (textContent === 'false') return false; - - // Return as string - return textContent; - } - - /** - * Synchronize UI state after successful tool execution - * Uses the centralized refresh method from the project store - */ - private syncUIState(): void { - try { - // Use the centralized refresh method from the store - const storeActions = useProjectStore.getState(); - if (storeActions.refreshProjectState) { - storeActions.refreshProjectState(); - } - - } catch (error) { - console.warn('Failed to sync UI state after XML tool execution:', error); - // Don't throw - UI sync failure shouldn't break tool execution - } - } - -} \ No newline at end of file diff --git a/src/agent/llm/ClaudeOpenRouterProvider.ts b/src/agent/llm/ClaudeOpenRouterProvider.ts deleted file mode 100644 index b630ca8..0000000 --- a/src/agent/llm/ClaudeOpenRouterProvider.ts +++ /dev/null @@ -1,239 +0,0 @@ -import { LLMProvider } from './LLMProvider'; -import type { StreamChunk } from './StreamingTypes'; -import type { Message } from '../core/AgentState'; -import { ConfigManager } from '../../core/config/ConfigManager'; -import { LLM_PROTOCOL } from '../../constants/llmConstants'; - -/** - * Claude (via OpenRouter) provider using the OpenAI-compatible Chat Completions API. - * Difference from the generic OpenAI provider: message content is an array of parts - * with a single text item per message (future-ready for images, tools, etc.). - */ -export class ClaudeOpenRouterProvider extends LLMProvider { - readonly name = 'Claude (OpenRouter)'; - - private isOllamaFormat: boolean | null = null; // Detected at runtime - - constructor() { - super(); - } - - /** - * Build OpenAI-compatible messages array where each message content is an array of parts. - */ - private buildRequestMessages(messages: Message[], systemPrompt?: string): Array<{ role: string; content: Array<{ type: 'text'; text: string; cache_control?: { type: 'ephemeral' } }> }> { - type ORPart = { type: 'text'; text: string; cache_control?: { type: 'ephemeral' } }; - const openAIMessages: Array<{ role: string; content: Array }> = []; - - // Add system prompt if provided - if (systemPrompt) { - openAIMessages.push({ role: 'system', content: [{ type: 'text', text: systemPrompt, cache_control: { type: 'ephemeral' } }] }); - } - - // Add conversation history with preserved roles - let lastUserIndex = -1; - for (let i = 0; i < messages.length; i++) { - if (messages[i].role === 'user') lastUserIndex = i; - } - - openAIMessages.push( - ...messages.map((msg, i) => { - const part: ORPart = { type: 'text', text: msg.content }; - if (msg.role === 'user' && i === lastUserIndex) { - part.cache_control = { type: 'ephemeral' }; - } - return { - role: msg.role, - content: [part] - }; - }) - ); - - return openAIMessages; - } - - /** - * Create API request with proper headers and body - */ - private async createApiRequest(messages: Array<{ role: string; content: Array<{ type: 'text'; text: string; cache_control?: { type: 'ephemeral' } }> }>, config: ReturnType, streaming: boolean): Promise { - const headers: Record = { - 'Authorization': `Bearer ${config.apiKey}`, - 'Content-Type': 'application/json', - }; - - // Optional, but recommended by OpenRouter docs to set referer/title for attribution - // if (typeof window !== 'undefined') { - // headers['HTTP-Referer'] = window.location.origin; - // headers['X-Title'] = 'K.G.Studio'; - // } - - const response = await fetch(config.apiEndpoint, { - method: 'POST', - headers, - body: JSON.stringify({ - model: config.model, - messages, - stream: streaming, - }), - }); - - if (!response.ok) { - const errorText = await response.text().catch(() => 'Unknown error'); - throw new Error(`${this.name} API request failed (${response.status}): ${response.statusText}. ${errorText}`); - } - - return response; - } - - /** - * Get current configuration values from ConfigManager - */ - private getCurrentConfig() { - const configManager = ConfigManager.instance(); - const apiKey = configManager.get('general.claude_openrouter.api_key') as string; - const model = configManager.get('general.claude_openrouter.model') as string; - const baseURL = configManager.get('general.claude_openrouter.base_url') as string; - // baseURL is the full API endpoint for OpenRouter (e.g., https://openrouter.ai/api/v1/chat/completions) - const apiEndpoint = baseURL; - return { apiKey, model, baseURL, apiEndpoint }; - } - - /** - * Detect if the response uses Ollama's raw JSON format or OpenAI's SSE format - */ - private detectStreamFormat(firstChunk: string): boolean { - // If it starts with "data: ", it's OpenAI SSE format - if (firstChunk.trim().startsWith(LLM_PROTOCOL.SSE_DATA_PREFIX)) { - return false; // Not Ollama format - } - // Try to parse as JSON - if successful and has 'done' field, it's Ollama format - try { - const json = JSON.parse(firstChunk.trim()); - return typeof json.done === 'boolean'; - } catch { - return false; // Not valid JSON, assume OpenAI format - } - } - - /** - * Parse Ollama's raw JSON chunk format - */ - private parseOllamaChunk(chunk: string): { thinking?: string; content?: string; isDone?: boolean } { - try { - const json = JSON.parse(chunk.trim()); - const thinking: string | undefined = json.message?.thinking; - const content: string | undefined = json.message?.content || json.response; // Handle both chat and completion formats - return { - thinking, - content, - isDone: json.done === true - }; - } catch { - return {}; // Invalid JSON, return empty object - } - } - - /** - * Parse OpenAI's SSE format chunk - */ - private parseOpenAIChunk(line: string): { thinking?: string; content?: string; isDone?: boolean } { - if (!line.startsWith(LLM_PROTOCOL.SSE_DATA_PREFIX)) { - return {}; - } - - const data = line.slice(LLM_PROTOCOL.SSE_DATA_PREFIX.length); - if (data === LLM_PROTOCOL.SSE_DONE_MARKER) { - return { isDone: true }; - } - - try { - const json = JSON.parse(data); - const delta = json.choices?.[0]?.delta; - const thinking: string | undefined = delta?.thinking; // Some providers may stream "thinking" - const content: string | undefined = delta?.content; - return { thinking, content, isDone: false }; - } catch { - return {}; // Skip invalid JSON lines - } - } - - private async *processContentChunk( - thinking: string | undefined, - content: string | undefined, - lastSegmentType: { current: 'thinking' | 'content' | null } - ): AsyncIterableIterator { - if (typeof thinking === 'string' && thinking.length > 0) { - if (lastSegmentType.current && lastSegmentType.current !== 'thinking') { - yield { type: 'text', content: LLM_PROTOCOL.SEGMENT_SEPARATOR }; - } - yield { type: 'text', content: thinking }; - lastSegmentType.current = 'thinking'; - } - - if (typeof content === 'string' && content.length > 0) { - if (lastSegmentType.current && lastSegmentType.current !== 'content') { - yield { type: 'text', content: LLM_PROTOCOL.SEGMENT_SEPARATOR }; - } - yield { type: 'text', content: content }; - lastSegmentType.current = 'content'; - } - } - - async *generateStream( - messages: Message[], - systemPrompt?: string - ): AsyncIterableIterator { - const config = this.getCurrentConfig(); - const requestMessages = this.buildRequestMessages(messages, systemPrompt); - const response = await this.createApiRequest(requestMessages, config, true); - - const reader = response.body?.getReader(); - if (!reader) { - throw new Error(`${this.name} streaming: Failed to get response reader from API response`); - } - - const decoder = new TextDecoder(); - let buffer = ''; - let firstChunkProcessed = false; - const lastSegmentType = { current: null as 'thinking' | 'content' | null }; - - try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - - buffer += decoder.decode(value, { stream: true }); - - // Split by newlines for SSE (and also works for line-delimited JSON) - const lines = buffer.split('\n'); - buffer = lines.pop() || ''; - - for (const line of lines) { - const trimmedLine = line.trim(); - if (!trimmedLine) continue; - - // Detect format on first non-empty chunk - if (!firstChunkProcessed) { - this.isOllamaFormat = this.detectStreamFormat(trimmedLine); - firstChunkProcessed = true; - } - - const { thinking, content, isDone } = this.isOllamaFormat - ? this.parseOllamaChunk(trimmedLine) - : this.parseOpenAIChunk(trimmedLine); - - if (isDone) { - yield { type: 'done', content: '' }; - return; - } - - yield* this.processContentChunk(thinking, content, lastSegmentType); - } - } - } finally { - reader.releaseLock(); - } - } -} - - diff --git a/src/agent/llm/ClaudeProvider.ts b/src/agent/llm/ClaudeProvider.ts deleted file mode 100644 index e9ec58a..0000000 --- a/src/agent/llm/ClaudeProvider.ts +++ /dev/null @@ -1,162 +0,0 @@ -import { LLMProvider } from './LLMProvider'; -import type { StreamChunk } from './StreamingTypes'; -import type { Message } from '../core/AgentState'; -import { ConfigManager } from '../../core/config/ConfigManager'; - -/** - * Anthropic Claude API provider implementation - */ -export class ClaudeProvider extends LLMProvider { - readonly name = 'Claude'; - - private apiKey: string; - private model: string; - private baseURL: string = 'https://api.anthropic.com'; - private apiEndpoint: string; - - constructor() { - super(); - - const configManager = ConfigManager.instance(); - this.apiKey = configManager.get('general.claude.api_key') as string; - this.model = configManager.get('general.claude.model') as string; - this.apiEndpoint = `${this.baseURL}/v1/messages`; - } - - /** - * Convert internal messages to Claude's format - */ - private convertMessages(messages: Message[], systemPrompt?: string): { - system?: string; - messages: Array<{ role: 'user' | 'assistant'; content: string }>; - } { - const claudeMessages: Array<{ role: 'user' | 'assistant'; content: string }> = []; - - claudeMessages.push(...messages.map(msg => ({ role: msg.role, content: msg.content }))); - - return { - system: systemPrompt, - messages: claudeMessages - }; - } - - /** - * Parse Claude's streaming response chunks - */ - private parseClaudeStreamChunk(line: string): { content?: string; isDone?: boolean } { - if (!line.startsWith('data: ')) { - return {}; - } - - const data = line.slice(6); - if (data === '[DONE]') { - return { isDone: true }; - } - - try { - const json = JSON.parse(data); - - // Handle different Claude streaming event types - switch (json.type) { - case 'content_block_delta': - return { - content: json.delta?.text, - isDone: false - }; - case 'message_stop': - return { isDone: true }; - default: - return {}; - } - } catch { - return {}; // Skip invalid JSON lines - } - } - - async *generateStream( - messages: Message[], - systemPrompt?: string, - tools?: Record[] - ): AsyncIterableIterator { - const { system, messages: claudeMessages } = this.convertMessages(messages, systemPrompt); - - const requestBody: { - model: string; - max_tokens: number; - messages: Array<{ role: 'user' | 'assistant'; content: string }>; - stream: boolean; - system?: string; - tools?: Record[]; - } = { - model: this.model, - max_tokens: 8192, - messages: claudeMessages, - stream: true - }; - - if (system) { - requestBody.system = system; - } - - if (tools && tools.length > 0) { - requestBody.tools = tools; - } - - const response = await fetch(this.apiEndpoint, { - method: 'POST', - headers: { - 'x-api-key': this.apiKey, - 'Content-Type': 'application/json', - 'anthropic-version': '2023-06-01' - }, - body: JSON.stringify(requestBody), - mode: 'cors' - }); - - if (!response.ok) { - throw new Error(`Claude API error: ${response.status} ${response.statusText}`); - } - - const reader = response.body?.getReader(); - if (!reader) { - throw new Error('Failed to get response reader'); - } - - const decoder = new TextDecoder(); - let buffer = ''; - - try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - - buffer += decoder.decode(value, { stream: true }); - - const lines = buffer.split('\n'); - buffer = lines.pop() || ''; - - for (const line of lines) { - const trimmedLine = line.trim(); - if (!trimmedLine) continue; - - const parseResult = this.parseClaudeStreamChunk(trimmedLine); - - if (parseResult.isDone) { - yield { type: 'done', content: '' }; - return; - } - - if (parseResult.content) { - yield { - type: 'text', - content: parseResult.content - }; - } - } - } - } finally { - reader.releaseLock(); - } - } - -} \ No newline at end of file diff --git a/src/agent/llm/GeminiProvider.ts b/src/agent/llm/GeminiProvider.ts deleted file mode 100644 index c62f0c5..0000000 --- a/src/agent/llm/GeminiProvider.ts +++ /dev/null @@ -1,183 +0,0 @@ -import { LLMProvider } from './LLMProvider'; -import type { StreamChunk } from './StreamingTypes'; -import type { Message } from '../core/AgentState'; -import { ConfigManager } from '../../core/config/ConfigManager'; - -/** - * Google Gemini API provider implementation - */ -export class GeminiProvider extends LLMProvider { - readonly name = 'Gemini'; - - private apiKey: string; - private model: string; - private baseURL: string = 'https://generativelanguage.googleapis.com'; - private apiEndpoint: string; - - constructor() { - super(); - - const configManager = ConfigManager.instance(); - this.apiKey = configManager.get('general.gemini.api_key') as string; - this.model = configManager.get('general.gemini.model') as string; - this.apiEndpoint = `${this.baseURL}/v1beta/models/${this.model}`; - } - - /** - * Convert internal messages to Gemini's format - */ - private convertMessages(messages: Message[], systemPrompt?: string): { - systemInstruction?: { parts: Array<{ text: string }> }; - contents: Array<{ role: 'user' | 'model'; parts: Array<{ text: string }> }>; - } { - const geminiMessages: Array<{ role: 'user' | 'model'; parts: Array<{ text: string }> }> = []; - - for (const msg of messages) { - let role: 'user' | 'model'; - - if (msg.role === 'assistant') { - role = 'model'; - } else { - // Treat system and user messages as 'user' role - role = 'user'; - } - - geminiMessages.push({ - role, - parts: [{ text: msg.content }] - }); - } - - const result: { - systemInstruction?: { parts: Array<{ text: string }> }; - contents: Array<{ role: 'user' | 'model'; parts: Array<{ text: string }> }>; - } = { - contents: geminiMessages - }; - - if (systemPrompt) { - result.systemInstruction = { - parts: [{ text: systemPrompt }] - }; - } - - return result; - } - - /** - * Parse Gemini's streaming response chunks - */ - private parseGeminiStreamChunk(chunk: string): { content?: string; isDone?: boolean } { - try { - const json = JSON.parse(chunk.trim()); - - // Gemini streaming format - if (json.candidates && json.candidates.length > 0) { - const candidate = json.candidates[0]; - - // Check if generation is finished - if (candidate.finishReason && candidate.finishReason !== 'STOP') { - return { isDone: true }; - } - - // Extract text content - const content = candidate.content?.parts?.[0]?.text; - if (content) { - return { content, isDone: false }; - } - } - - // Check for explicit done signal - if (json.done === true) { - return { isDone: true }; - } - - return {}; - } catch { - return {}; // Skip invalid JSON - } - } - - async *generateStream( - messages: Message[], - systemPrompt?: string, - tools?: Record[] - ): AsyncIterableIterator { - const { systemInstruction, contents } = this.convertMessages(messages, systemPrompt); - - const requestBody: { - contents: Array<{ role: 'user' | 'model'; parts: Array<{ text: string }> }>; - generationConfig: { temperature: number; maxOutputTokens: number }; - systemInstruction?: { parts: Array<{ text: string }> }; - tools?: Record[]; - } = { - contents, - generationConfig: { - temperature: 0.7, - maxOutputTokens: 8192, - } - }; - - if (systemInstruction) { - requestBody.systemInstruction = systemInstruction; - } - - if (tools && tools.length > 0) { - requestBody.tools = tools; - } - - const response = await fetch(`${this.apiEndpoint}:streamGenerateContent?key=${this.apiKey}`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(requestBody), - }); - - if (!response.ok) { - throw new Error(`Gemini API error: ${response.status} ${response.statusText}`); - } - - const reader = response.body?.getReader(); - if (!reader) { - throw new Error('Failed to get response reader'); - } - - const decoder = new TextDecoder(); - let buffer = ''; - - try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - - buffer += decoder.decode(value, { stream: true }); - - // Gemini sends JSON objects separated by newlines - const lines = buffer.split('\n'); - buffer = lines.pop() || ''; - - for (const line of lines) { - const trimmedLine = line.trim(); - if (!trimmedLine) continue; - - const parseResult = this.parseGeminiStreamChunk(trimmedLine); - - if (parseResult.isDone) { - yield { type: 'done', content: '' }; - return; - } - - if (parseResult.content) { - yield { - type: 'text', - content: parseResult.content - }; - } - } - } - } finally { - reader.releaseLock(); - } - } -} \ No newline at end of file diff --git a/src/agent/llm/LLMProvider.ts b/src/agent/llm/LLMProvider.ts index 71d78f0..ad896e8 100644 --- a/src/agent/llm/LLMProvider.ts +++ b/src/agent/llm/LLMProvider.ts @@ -1,21 +1,146 @@ +import OpenAI from 'openai'; import type { StreamChunk } from './StreamingTypes'; -import type { Message } from '../core/AgentState'; +import type { Message, ToolCall } from '../core/AgentState'; +import type { OpenAIToolDefinition } from '../tools/BaseTool'; /** - * Abstract interface for LLM providers + * LLM provider using the OpenAI SDK. + * Works with any OpenAI-compatible API (OpenAI, OpenRouter, Ollama, vLLM, etc.) */ -export abstract class LLMProvider { - abstract name: string; - +export class LLMProvider { + private client: OpenAI; + private model: string; + + constructor(apiKey: string, model: string, baseURL?: string) { + // The OpenAI SDK appends /chat/completions itself, so strip it if the user included it + const normalizedBaseURL = baseURL?.replace(/\/chat\/completions\/?$/, '') || undefined; + + this.client = new OpenAI({ + apiKey, + ...(normalizedBaseURL ? { baseURL: normalizedBaseURL } : {}), + dangerouslyAllowBrowser: true, + }); + this.model = model; + } + /** - * Generate a streaming response from the LLM - * @param messages The full conversation history with preserved roles - * @param systemPrompt The system prompt (optional, can be included in messages) - * @param tools Available tools (optional for now) + * Convert internal Message[] to OpenAI ChatCompletionMessageParam[] */ - abstract generateStream( - messages: Message[], + private convertMessages( + messages: Message[], + systemPrompt?: string + ): OpenAI.ChatCompletionMessageParam[] { + const result: OpenAI.ChatCompletionMessageParam[] = []; + + if (systemPrompt) { + result.push({ role: 'system', content: systemPrompt }); + } + + for (const msg of messages) { + if (msg.role === 'user') { + result.push({ role: 'user', content: msg.content ?? '' }); + } else if (msg.role === 'assistant') { + const assistantMsg: OpenAI.ChatCompletionAssistantMessageParam = { + role: 'assistant', + content: msg.content ?? null, + }; + if (msg.tool_calls && msg.tool_calls.length > 0) { + assistantMsg.tool_calls = msg.tool_calls.map(tc => ({ + id: tc.id, + type: 'function' as const, + function: { name: tc.function.name, arguments: tc.function.arguments }, + })); + } + result.push(assistantMsg); + } else if (msg.role === 'tool') { + result.push({ + role: 'tool', + tool_call_id: msg.tool_call_id!, + content: msg.content ?? '', + }); + } + } + + return result; + } + + /** + * Generate a streaming response from the LLM. + * Yields StreamChunks for text content and tool calls. + */ + async *generateStream( + messages: Message[], systemPrompt?: string, - tools?: Record[] - ): AsyncIterableIterator; -} \ No newline at end of file + tools?: OpenAIToolDefinition[], + ): AsyncIterableIterator { + const openaiMessages = this.convertMessages(messages, systemPrompt); + + const requestParams: OpenAI.ChatCompletionCreateParamsStreaming = { + model: this.model, + messages: openaiMessages, + stream: true, + }; + + if (tools && tools.length > 0) { + requestParams.tools = tools as unknown as OpenAI.ChatCompletionTool[]; + requestParams.tool_choice = 'auto'; + } + + const stream = this.client.chat.completions.stream(requestParams); + + // Accumulate tool calls across chunks (they arrive incrementally) + const toolCallAccumulator = new Map(); + + for await (const chunk of stream) { + console.log('LLMProvider: chunk', JSON.stringify(chunk)); + const choice = chunk.choices[0]; + if (!choice) continue; + + const delta = choice.delta; + + // Yield text content + if (delta.content) { + yield { type: 'text', content: delta.content }; + } + + // Accumulate tool calls from deltas + if (delta.tool_calls) { + for (const tc of delta.tool_calls) { + const existing = toolCallAccumulator.get(tc.index); + if (existing) { + // Append to existing tool call + if (tc.function?.arguments) { + existing.arguments += tc.function.arguments; + } + } else { + // New tool call + toolCallAccumulator.set(tc.index, { + id: tc.id ?? '', + name: tc.function?.name ?? '', + arguments: tc.function?.arguments ?? '', + }); + } + } + } + } + + // After stream ends, get the final completion for finish_reason + const finalCompletion = await stream.finalChatCompletion(); + const finishReason = finalCompletion.choices[0]?.finish_reason ?? 'stop'; + + // Emit accumulated tool calls + if (toolCallAccumulator.size > 0) { + for (const [, tc] of toolCallAccumulator) { + const toolCall: ToolCall = { + id: tc.id, + type: 'function', + function: { name: tc.name, arguments: tc.arguments }, + }; + yield { type: 'tool_call', content: '', toolCall }; + } + } + + // Signal completion + yield { type: 'done', content: '', finishReason }; + } +} diff --git a/src/agent/llm/OpenAIProvider.ts b/src/agent/llm/OpenAIProvider.ts deleted file mode 100644 index 0a36ad7..0000000 --- a/src/agent/llm/OpenAIProvider.ts +++ /dev/null @@ -1,237 +0,0 @@ -import { LLMProvider } from './LLMProvider'; -import type { StreamChunk } from './StreamingTypes'; -import type { Message } from '../core/AgentState'; -import { ConfigManager } from '../../core/config/ConfigManager'; -import { URL_CONSTANTS } from '../../constants/coreConstants'; -import { LLM_PROTOCOL } from '../../constants/llmConstants'; - -/** - * OpenAI API provider implementation - */ -export class OpenAIProvider extends LLMProvider { - readonly name = 'OpenAI'; - - private isOllamaFormat: boolean | null = null; // Detected at runtime - - constructor() { - super(); - } - - /** - * Build OpenAI-compatible messages array from input messages and system prompt - */ - private buildRequestMessages(messages: Message[], systemPrompt?: string): Array<{ role: string; content: string }> { - const openAIMessages: Array<{ role: string; content: string }> = []; - - // Add system prompt if provided - if (systemPrompt) { - openAIMessages.push({ role: 'system', content: systemPrompt }); - } - - // Add conversation history with preserved roles - openAIMessages.push(...messages.map(msg => ({ - role: msg.role, - content: msg.content - }))); - - return openAIMessages; - } - - /** - * Create API request with proper headers and body - */ - private async createApiRequest(messages: Array<{ role: string; content: string }>, config: ReturnType, streaming: boolean): Promise { - const response = await fetch(config.apiEndpoint, { - method: 'POST', - headers: { - 'Authorization': `Bearer ${config.apiKey}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - model: config.model, - ...(config.flexMode && !config.isCompatibleProvider ? { service_tier: 'flex' } : {}), - messages, - stream: streaming, - }), - }); - - if (!response.ok) { - const errorText = await response.text().catch(() => 'Unknown error'); - throw new Error(`${this.name} API request failed (${response.status}): ${response.statusText}. ${errorText}`); - } - - return response; - } - - /** - * Process thinking and content chunks, yielding appropriate StreamChunks - */ - private async *processContentChunk( - thinking: string | undefined, - content: string | undefined, - lastSegmentType: { current: 'thinking' | 'content' | null } - ): AsyncIterableIterator { - if (typeof thinking === 'string' && thinking.length > 0) { - if (lastSegmentType.current && lastSegmentType.current !== 'thinking') { - yield { type: 'text', content: LLM_PROTOCOL.SEGMENT_SEPARATOR }; - } - yield { type: 'text', content: thinking }; - lastSegmentType.current = 'thinking'; - } - - if (typeof content === 'string' && content.length > 0) { - if (lastSegmentType.current && lastSegmentType.current !== 'content') { - yield { type: 'text', content: LLM_PROTOCOL.SEGMENT_SEPARATOR }; - } - yield { type: 'text', content: content }; - lastSegmentType.current = 'content'; - } - } - - - /** - * Get current configuration values from ConfigManager - */ - private getCurrentConfig() { - const configManager = ConfigManager.instance(); - const llmProvider = configManager.get('general.llm_provider') as string; - const isCompatibleProvider = llmProvider === 'openai_compatible'; - - if (isCompatibleProvider) { - const apiKey = configManager.get('general.openai_compatible.api_key') as string; - const model = configManager.get('general.openai_compatible.model') as string; - const baseURL = configManager.get('general.openai_compatible.base_url') as string; - // For compatible providers, use the full URL as provided (assume it includes the endpoint) - // Common patterns: http://localhost:11434/api/chat (Ollama), https://api.openrouter.ai/v1 (OpenRouter) - const apiEndpoint = baseURL; - const flexMode = false; // Not applicable to compatible providers - - return { apiKey, model, baseURL, apiEndpoint, flexMode, isCompatibleProvider }; - } else { - const apiKey = configManager.get('general.openai.api_key') as string; - const model = configManager.get('general.openai.model') as string; - const flexMode = (configManager.get('general.openai.flex') as boolean) === true; - const baseURL = URL_CONSTANTS.DEFAULT_OPENAI_BASE_URL; - const apiEndpoint = `${baseURL}/chat/completions`; - - return { apiKey, model, baseURL, apiEndpoint, flexMode, isCompatibleProvider }; - } - } - - /** - * Detect if the response uses Ollama's raw JSON format or OpenAI's SSE format - */ - private detectStreamFormat(firstChunk: string): boolean { - // If it starts with "data: ", it's OpenAI SSE format - if (firstChunk.trim().startsWith(LLM_PROTOCOL.SSE_DATA_PREFIX)) { - return false; // Not Ollama format - } - - // Try to parse as JSON - if successful and has 'done' field, it's Ollama format - try { - const json = JSON.parse(firstChunk.trim()); - return typeof json.done === 'boolean'; - } catch { - return false; // Not valid JSON, assume OpenAI format - } - } - - /** - * Parse Ollama's raw JSON chunk format - */ - private parseOllamaChunk(chunk: string): { thinking?: string; content?: string; isDone?: boolean } { - try { - const json = JSON.parse(chunk.trim()); - const thinking: string | undefined = json.message?.thinking; - const content: string | undefined = json.message?.content || json.response; // Handle both chat and completion formats - return { - thinking, - content, - isDone: json.done === true - }; - } catch { - return {}; // Invalid JSON, return empty object - } - } - - /** - * Parse OpenAI's SSE format chunk - */ - private parseOpenAIChunk(line: string): { thinking?: string; content?: string; isDone?: boolean } { - if (!line.startsWith(LLM_PROTOCOL.SSE_DATA_PREFIX)) { - return {}; - } - - const data = line.slice(LLM_PROTOCOL.SSE_DATA_PREFIX.length); - if (data === LLM_PROTOCOL.SSE_DONE_MARKER) { - return { isDone: true }; - } - - try { - const json = JSON.parse(data); - const delta = json.choices?.[0]?.delta; - const thinking: string | undefined = delta?.thinking; // Some providers may stream "thinking" - const content: string | undefined = delta?.content; - return { thinking, content, isDone: false }; - } catch { - return {}; // Skip invalid JSON lines - } - } - - async *generateStream( - messages: Message[], - systemPrompt?: string - ): AsyncIterableIterator { - const config = this.getCurrentConfig(); - const requestMessages = this.buildRequestMessages(messages, systemPrompt); - const response = await this.createApiRequest(requestMessages, config, true); - - const reader = response.body?.getReader(); - if (!reader) { - throw new Error(`${this.name} streaming: Failed to get response reader from API response`); - } - - const decoder = new TextDecoder(); - let buffer = ''; - let firstChunkProcessed = false; - const lastSegmentType = { current: null as 'thinking' | 'content' | null }; - - try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - - buffer += decoder.decode(value, { stream: true }); - - // For Ollama format, we need to split by newlines for JSON objects - // For OpenAI format, we also split by newlines for SSE - const lines = buffer.split('\n'); - buffer = lines.pop() || ''; - - for (const line of lines) { - const trimmedLine = line.trim(); - if (!trimmedLine) continue; - - // Detect format on first non-empty chunk - if (!firstChunkProcessed) { - this.isOllamaFormat = this.detectStreamFormat(trimmedLine); - firstChunkProcessed = true; - } - - const { thinking, content, isDone } = this.isOllamaFormat - ? this.parseOllamaChunk(trimmedLine) - : this.parseOpenAIChunk(trimmedLine); - - if (isDone) { - yield { type: 'done', content: '' }; - return; - } - - yield* this.processContentChunk(thinking, content, lastSegmentType); - } - } - } finally { - reader.releaseLock(); - } - } -} \ No newline at end of file diff --git a/src/agent/llm/StreamingTypes.ts b/src/agent/llm/StreamingTypes.ts index d53c4d1..738ca01 100644 --- a/src/agent/llm/StreamingTypes.ts +++ b/src/agent/llm/StreamingTypes.ts @@ -1,27 +1,13 @@ /** - * Types for streaming LLM responses and tool execution + * Types for streaming LLM responses */ -import type { ToolResult } from '../tools/BaseTool'; - -// Re-export for convenience -export type { ToolResult }; - -export interface ToolInvocation { - id: string; - name: string; - parameters: Record; -} +import type { ToolCall } from '../core/AgentState'; export interface StreamChunk { type: 'text' | 'tool_call' | 'tool_result' | 'done'; content: string; - toolCall?: ToolInvocation; - toolResult?: ToolResult; + toolCall?: ToolCall; + toolResult?: { name: string; success: boolean; result: string }; + finishReason?: string; // 'stop' | 'tool_calls' — present on 'done' chunks } - -export interface LLMResponse { - content: string; - toolCalls?: ToolInvocation[]; - finished: boolean; -} \ No newline at end of file diff --git a/src/agent/tools/AddNotesTool.ts b/src/agent/tools/AddNotesTool.ts index f5de003..df9ee08 100644 --- a/src/agent/tools/AddNotesTool.ts +++ b/src/agent/tools/AddNotesTool.ts @@ -12,35 +12,35 @@ import { KGCore } from '../../core/KGCore'; */ export class AddNotesTool extends BaseTool { readonly name = 'add_notes'; - readonly description = 'Create one or more MIDI notes in the current region. Each note requires pitch (e.g., "C4", "F#3"), start_beat (beat position), and length (duration in beats).'; - + readonly description = 'Add one or more MIDI notes to the current region. Use this to create melodies, chords, or any musical content. Notes use absolute beat positions on the project timeline — not relative to the region start.'; + readonly parameters: Record = { notes: { type: 'array', - description: 'Array of notes to create', + description: 'List of notes to add. To create a chord, give multiple notes the same start_beat. To create a melody, use sequential start_beat values.', required: true, items: { type: 'object', - description: 'A MIDI note definition', + description: 'A single note', properties: { pitch: { type: 'string', - description: 'Note pitch in scientific notation (e.g., "C4", "F#3", "Bb2")', + description: 'Pitch in scientific notation: note name, optional accidental (# or b), and octave number. Examples: "C4" (middle C), "F#3" (F-sharp 3rd octave), "Bb2" (B-flat 2nd octave).', required: true }, start_beat: { type: 'number', - description: 'Start position in beats (e.g., 0, 1.5, 2)', + description: 'Absolute beat position on the project timeline where the note starts. This is NOT relative to the region — beat 6 means beat 6 in the project regardless of where the region begins. Fractional values are supported (e.g., 0.5 = half a beat after beat 0).', required: true }, length: { type: 'number', - description: 'Note duration in beats (e.g., 1, 0.5, 4)', + description: 'Duration of the note in beats. In 4/4 time: 4 = whole note, 2 = half note, 1 = quarter note, 0.5 = eighth note, 0.25 = sixteenth note.', required: true }, velocity: { type: 'number', - description: 'Note velocity (1-127, default: 127)', + description: 'Note velocity / loudness from 1 (softest) to 127 (loudest). Defaults to 127 if omitted.', required: false } } @@ -48,7 +48,7 @@ export class AddNotesTool extends BaseTool { }, region_id: { type: 'string', - description: 'ID of the region to add notes to. If not provided, uses the currently selected region.', + description: 'Target region ID. If omitted, uses the currently active piano roll region or selected region.', required: false } }; diff --git a/src/agent/tools/AttemptCompletionTool.ts b/src/agent/tools/AttemptCompletionTool.ts deleted file mode 100644 index 8e2f271..0000000 --- a/src/agent/tools/AttemptCompletionTool.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { BaseTool } from './BaseTool'; -import type { ToolResult, ToolParameter } from './BaseTool'; -import { AgentCore } from '../core/AgentCore'; - -/** - * Tool for signaling task completion - * This is a pure agent state tool that doesn't modify the DAW but signals - * to the agent system that the user's requested task has been completed - */ -export class AttemptCompletionTool extends BaseTool { - readonly name = 'attempt_completion'; - readonly description = 'Signal that the current user task is fully complete. Only use this when you have successfully fulfilled all aspects of the user\'s request.'; - - readonly parameters: Record = { - comment: { - type: 'string', - description: 'A brief comment describing what was completed and any relevant details about the task fulfillment.', - required: true - } - }; - - async execute(params: Record): Promise { - try { - // Validate parameters - this.validateParameters(params); - - const comment = params.comment as string; - - // Validate comment is not empty - if (!comment.trim()) { - return this.createErrorResult('Comment cannot be empty. Please provide a meaningful completion summary.'); - } - - // Get current agent state and update task completion status - const agentCore = AgentCore.instance(); - const agentState = agentCore.getAgentState(); - - // Mark that we're no longer working on a task - agentState.setIsWorkingOnTask(false); - - return this.createSuccessResult( - `Task completed: ${comment}. ` - ); - - } catch (error) { - return this.createErrorResult(`Failed to mark task as complete: ${error}`); - } - } -} \ No newline at end of file diff --git a/src/agent/tools/BaseTool.ts b/src/agent/tools/BaseTool.ts index 4f3215f..7aa5d94 100644 --- a/src/agent/tools/BaseTool.ts +++ b/src/agent/tools/BaseTool.ts @@ -21,7 +21,28 @@ export interface ToolParameter { } /** - * Tool definition schema + * OpenAI-compatible JSON Schema for function parameters + */ +export interface OpenAIFunctionParameters { + type: 'object'; + properties: Record; + required?: string[]; +} + +/** + * OpenAI-compatible tool definition + */ +export interface OpenAIToolDefinition { + type: 'function'; + function: { + name: string; + description: string; + parameters: OpenAIFunctionParameters; + }; +} + +/** + * Tool definition schema (internal format) */ export interface ToolDefinition { name: string; @@ -48,14 +69,71 @@ export abstract class BaseTool { /** * Get the tool definition in OpenAI function calling format */ - getDefinition(): ToolDefinition { + getDefinition(): OpenAIToolDefinition { return { - name: this.name, - description: this.description, - parameters: this.parameters + type: 'function', + function: { + name: this.name, + description: this.description, + parameters: this.convertToJsonSchema(this.parameters) + } }; } + /** + * Convert internal ToolParameter map to OpenAI-compatible JSON Schema + */ + private convertToJsonSchema(params: Record): OpenAIFunctionParameters { + const properties: Record = {}; + const required: string[] = []; + + for (const [name, param] of Object.entries(params)) { + properties[name] = this.convertParamToJsonSchema(param); + if (param.required) { + required.push(name); + } + } + + return { + type: 'object', + properties, + ...(required.length > 0 ? { required } : {}) + }; + } + + /** + * Convert a single ToolParameter to JSON Schema format + */ + private convertParamToJsonSchema(param: ToolParameter): Record { + const schema: Record = { + type: param.type, + description: param.description + }; + + if (param.type === 'array' && param.items) { + schema.items = this.convertParamToJsonSchema(param.items); + } + + if (param.type === 'object' && param.properties) { + const properties: Record = {}; + const required: string[] = []; + + for (const [name, prop] of Object.entries(param.properties)) { + properties[name] = this.convertParamToJsonSchema(prop); + if (prop.required) { + required.push(name); + } + } + + schema.properties = properties; + if (required.length > 0) { + schema.required = required; + } + } + + return schema; + } + /** * Validate parameters against the tool's parameter schema * @param params Parameters to validate diff --git a/src/agent/tools/ReadMusicTool.ts b/src/agent/tools/ReadMusicTool.ts index cf39a80..eafd395 100644 --- a/src/agent/tools/ReadMusicTool.ts +++ b/src/agent/tools/ReadMusicTool.ts @@ -12,22 +12,22 @@ import { FLUIDR3_INSTRUMENT_MAP } from '../../constants/generalMidiConstants'; */ export class ReadMusicTool extends BaseTool { readonly name = 'read_music'; - readonly description = 'Read the music content from a specific track or all tracks, returning the content in ABC notation format.'; - + readonly description = 'Read existing musical content from one or more tracks, returned as ABC notation. Use this to understand what notes already exist before making edits. Always call this before asking the user about their music. The output is bar-aligned and includes key/time signature headers.'; + readonly parameters: Record = { track_id: { type: 'string', - description: 'The track ID to read, or "all" to read all tracks. If not provided, reads the first available track.', + description: 'Which track to read. Pass a specific track ID, or "all" to read every track. If omitted, reads the first available track.', required: false }, start_beat: { type: 'number', - description: 'Start beat position to read from (default: 0)', + description: 'Absolute beat position to start reading from. The actual output will be rounded down to the nearest bar boundary. Defaults to 0.', required: false }, length: { type: 'number', - description: 'Length in beats to read (default: entire track/project)', + description: 'Number of beats to read. The actual output will be rounded up to the nearest bar boundary. If omitted, reads to the end of the track.', required: false } }; diff --git a/src/agent/tools/RemoveNotesTool.ts b/src/agent/tools/RemoveNotesTool.ts index ce2c774..9f3147e 100644 --- a/src/agent/tools/RemoveNotesTool.ts +++ b/src/agent/tools/RemoveNotesTool.ts @@ -11,22 +11,22 @@ import { KGCore } from '../../core/KGCore'; */ export class RemoveNotesTool extends BaseTool { readonly name = 'remove_notes'; - readonly description = 'Remove MIDI notes from the current region within a specified beat range. All notes that start within the range will be deleted.'; - + readonly description = 'Remove all MIDI notes whose start position falls within the specified beat range. Use this to clear a section before rewriting it, or to delete unwanted notes. Beat positions are absolute on the project timeline.'; + readonly parameters: Record = { start_beat: { type: 'number', - description: 'Start of the beat range to remove notes from (inclusive)', + description: 'Absolute beat position where the removal range begins (inclusive). A note starting at exactly this beat will be removed.', required: true }, end_beat: { type: 'number', - description: 'End of the beat range to remove notes from (exclusive)', + description: 'Absolute beat position where the removal range ends (exclusive). A note starting at exactly this beat will NOT be removed. Must be greater than start_beat.', required: true }, region_id: { type: 'string', - description: 'ID of the region to remove notes from. If not provided, uses the currently selected region.', + description: 'Target region ID. If omitted, uses the currently active piano roll region or selected region.', required: false } }; diff --git a/src/agent/tools/ThinkTool.ts b/src/agent/tools/ThinkTool.ts deleted file mode 100644 index 139e3c4..0000000 --- a/src/agent/tools/ThinkTool.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { BaseTool } from './BaseTool'; -import type { ToolResult, ToolParameter } from './BaseTool'; - -/** - * Pseudo tool for handling tags in LLM responses - * This tool displays the thinking content in the UI but doesn't send results back to the LLM - * Handles XML format: any content here - * This is functionally identical to ThinkingTool but handles the shorter tag name - */ -export class ThinkTool extends BaseTool { - readonly name = 'think'; - readonly description = 'Pseudo tool for handling LLM thinking content from tags. Shows content in UI but does not send results back to LLM.'; - - readonly parameters: Record = { - content: { - type: 'string', - description: 'The thinking content from the XML tag', - required: false - } - }; - - async execute(params: Record): Promise { - try { - // Extract the thinking content from the parameters - // const content = params.content as string || ''; - - // Return the thinking content as a successful result - // This will be displayed in the UI but not sent back to the LLM - return this.createSuccessResult("Thinking completed."); - - } catch (error) { - return this.createErrorResult(`Failed to process thinking content: ${error}`); - } - } -} \ No newline at end of file diff --git a/src/agent/tools/ThinkingTool.ts b/src/agent/tools/ThinkingTool.ts deleted file mode 100644 index a710b6c..0000000 --- a/src/agent/tools/ThinkingTool.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { BaseTool } from './BaseTool'; -import type { ToolResult, ToolParameter } from './BaseTool'; - -/** - * Pseudo tool for handling tags in LLM responses - * This tool displays the thinking content in the UI but doesn't send results back to the LLM - * Handles XML format: any content here - */ -export class ThinkingTool extends BaseTool { - readonly name = 'thinking'; - readonly description = 'Pseudo tool for handling LLM thinking content. Shows content in UI but does not send results back to LLM.'; - - readonly parameters: Record = { - content: { - type: 'string', - description: 'The thinking content from the XML tag', - required: false - } - }; - - async execute(params: Record): Promise { - try { - // Extract the thinking content from the parameters - // const content = params.content as string || ''; - - // Return the thinking content as a successful result - // This will be displayed in the UI but not sent back to the LLM - return this.createSuccessResult("Thinking completed."); - - } catch (error) { - return this.createErrorResult(`Failed to process thinking content: ${error}`); - } - } -} \ No newline at end of file diff --git a/src/agent/tools/index.ts b/src/agent/tools/index.ts index dd75d43..6c513ce 100644 --- a/src/agent/tools/index.ts +++ b/src/agent/tools/index.ts @@ -1,25 +1,19 @@ // Base tool system export { BaseTool } from './BaseTool'; -export type { ToolResult, ToolParameter, ToolDefinition } from './BaseTool'; +export type { ToolResult, ToolParameter, ToolDefinition, OpenAIToolDefinition, OpenAIFunctionParameters } from './BaseTool'; // Specific tools import { AddNotesTool } from './AddNotesTool'; import { RemoveNotesTool } from './RemoveNotesTool'; import { ReadMusicTool } from './ReadMusicTool'; -import { AttemptCompletionTool } from './AttemptCompletionTool'; -import { ThinkingTool } from './ThinkingTool'; -import { ThinkTool } from './ThinkTool'; -export { AddNotesTool, RemoveNotesTool, ReadMusicTool, AttemptCompletionTool, ThinkingTool, ThinkTool }; +export { AddNotesTool, RemoveNotesTool, ReadMusicTool }; // Tool registry for easy access export const AVAILABLE_TOOLS = { add_notes: AddNotesTool, remove_notes: RemoveNotesTool, read_music: ReadMusicTool, - attempt_completion: AttemptCompletionTool, - thinking: ThinkingTool, - think: ThinkTool } as const; -export type ToolName = keyof typeof AVAILABLE_TOOLS; \ No newline at end of file +export type ToolName = keyof typeof AVAILABLE_TOOLS; diff --git a/src/components/ChatBox.tsx b/src/components/ChatBox.tsx index 9eea23b..2a957bb 100644 --- a/src/components/ChatBox.tsx +++ b/src/components/ChatBox.tsx @@ -2,10 +2,6 @@ import React, { useState, useRef, useEffect, memo, useCallback } from 'react'; import { FaPlus, FaBan, FaDownload } from 'react-icons/fa'; import { UserMessage, AssistantMessage } from './chat'; import { AgentCore } from '../agent/core/AgentCore'; -import { OpenAIProvider } from '../agent/llm/OpenAIProvider'; -import { ClaudeProvider } from '../agent/llm/ClaudeProvider'; -import { ClaudeOpenRouterProvider } from '../agent/llm/ClaudeOpenRouterProvider'; -import { GeminiProvider } from '../agent/llm/GeminiProvider'; import { LLMProvider } from '../agent/llm/LLMProvider'; import { ConfigManager } from '../core/config/ConfigManager'; import { useProjectStore } from '../stores/projectStore'; @@ -14,10 +10,8 @@ import { clearChatHistoryAndUI, registerClearChatUICallback } from '../util/chat import { processUserMessage } from '../util/messageFilter/UserMessageFilter'; import { useStreamProcessor } from '../hooks/useStreamProcessor'; import { createMessage, addWelcomeMessage } from '../utils/chatMessageUtils'; -import { extractActionableTools, executeAllTools } from '../utils/toolExecutionUtils'; import { formatLocalDateTime } from '../util/timeUtil'; import { downloadBlob, buildTimestampSuffix } from '../util/miscUtil'; -import { wrapXmlBlocksInContent } from '../util/xmlUtil'; import KGDropdown from './common/KGDropdown'; import type { ChatMessage } from '../types/projectTypes'; @@ -26,24 +20,31 @@ import type { ChatMessage } from '../types/projectTypes'; let hasShownWelcomeOnceInRuntime = false; /** - * Create the appropriate LLM provider based on configuration + * Create the LLM provider from current configuration */ -const createLLMProvider = (): LLMProvider => { +const createLLMProviderFromConfig = (): LLMProvider => { const configManager = ConfigManager.instance(); const providerType = configManager.get('general.llm_provider') as string; - + + let apiKey: string; + let model: string; + let baseURL: string | undefined; + switch (providerType) { - case 'claude': - return new ClaudeProvider(); - case 'gemini': - return new GeminiProvider(); - case 'claude_openrouter': - return new ClaudeOpenRouterProvider(); - case 'openai_compatible': case 'openai': + apiKey = configManager.get('general.openai.api_key') as string; + model = configManager.get('general.openai.model') as string; + baseURL = undefined; // Uses OpenAI default + break; + case 'openai_compatible': default: - return new OpenAIProvider(); + apiKey = configManager.get('general.openai_compatible.api_key') as string; + model = configManager.get('general.openai_compatible.model') as string; + baseURL = configManager.get('general.openai_compatible.base_url') as string || undefined; + break; } + + return new LLMProvider(apiKey, model, baseURL); }; interface ChatBoxProps { @@ -53,17 +54,11 @@ interface ChatBoxProps { const ChatBox: React.FC = ({ isVisible }) => { const [inputValue, setInputValue] = useState(''); const textareaRef = useRef(null); - - // Initialize with empty messages + const [messages, setMessages] = useState([]); const [isProcessing, setIsProcessing] = useState(false); const [lastUserMessage, setLastUserMessage] = useState(''); - - // Tool execution state - const [isExecutingTools, setIsExecutingTools] = useState(false); - const [, setToolResults] = useState(''); // placeholder for future display/use - const [, setCurrentToolIndex] = useState(0); // placeholder for future display/use - + // Track if this is the first message (for system prompt logging) const [isFirstMessage, setIsFirstMessage] = useState(true); @@ -77,8 +72,8 @@ const ChatBox: React.FC = ({ isVisible }) => { const handleExportOptionSelect = (option: string) => { if (option === 'Export conversation as JSON') { try { - const messages = AgentCore.instance().getAgentState().getMessages(); - const exportMessages = messages.map((m) => ({ + const agentMessages = AgentCore.instance().getAgentState().getMessages(); + const exportMessages = agentMessages.map((m) => ({ id: m.id, role: m.role, content: m.content, @@ -93,25 +88,21 @@ const ChatBox: React.FC = ({ isVisible }) => { } else if (option === 'Export conversation as Markdown') { (async () => { try { - const messages = AgentCore.instance().getAgentState().getMessages(); + const agentMessages = AgentCore.instance().getAgentState().getMessages(); const templateUrl = `${import.meta.env.BASE_URL}chat/export_conversation_template.md`; const res = await fetch(templateUrl); const template = await res.text(); - const isAutomatedUserMessage = (content: string): boolean => { - return /^tool:\s.*\nsuccess:\s*(true|false)/i.test(content); - }; - - const sections = messages.map((m) => { - const isAutomaticUserMessage = isAutomatedUserMessage(m.content); - const roleLabel = m.role === 'assistant' ? 'Assistant' : (isAutomaticUserMessage ? 'User (Automatic)' : 'User'); - const ts = formatLocalDateTime(new Date(m.timestamp)); - const contentWithXml = isAutomaticUserMessage ? "```\n" + m.content + "\n```" : wrapXmlBlocksInContent(m.content); - return template - .replace('{role}', roleLabel) - .replace('{timestamp}', ts) - .replace('{content}', contentWithXml); - }); + const sections = agentMessages + .filter(m => m.role === 'user' || m.role === 'assistant') + .map((m) => { + const roleLabel = m.role === 'assistant' ? 'Assistant' : 'User'; + const ts = formatLocalDateTime(new Date(m.timestamp)); + return template + .replace('{role}', roleLabel) + .replace('{timestamp}', ts) + .replace('{content}', m.content ?? ''); + }); const markdown = sections.join('\n'); const filename = `kgstudio-conversation-${buildTimestampSuffix()}.md`; @@ -120,8 +111,6 @@ const ChatBox: React.FC = ({ isVisible }) => { console.error('Failed to export conversation as Markdown:', err); } })(); - } else { - console.log('Chat export selected:', option); } setShowExportDropdown(false); }; @@ -135,6 +124,10 @@ const ChatBox: React.FC = ({ isVisible }) => { setMessages(prev => [...prev, message]); }, []); + const handleMessageRemove = useCallback((messageId: string) => { + setMessages(prev => prev.filter(msg => msg.id !== messageId)); + }, []); + const handleProcessingChange = useCallback((processing: boolean) => { setIsProcessing(processing); }, []); @@ -143,17 +136,14 @@ const ChatBox: React.FC = ({ isVisible }) => { const streamProcessor = useStreamProcessor({ onMessageUpdate: handleMessageUpdate, onMessageAdd: handleMessageAdd, + onMessageRemove: handleMessageRemove, onProcessingChange: handleProcessingChange }); const clearChatUI = useCallback(async () => { - // Clear UI state setMessages([]); - - // Reset first message flag so system prompt will be logged again setIsFirstMessage(true); - - // Auto-show welcome message after clearing (like on app startup) + const welcomeMessage = await addWelcomeMessage(); if (welcomeMessage) { setMessages([welcomeMessage]); @@ -164,47 +154,37 @@ const ChatBox: React.FC = ({ isVisible }) => { useEffect(() => { const initializeProvider = async () => { const configManager = ConfigManager.instance(); - - // Ensure ConfigManager is initialized + if (!configManager.getIsInitialized()) { await configManager.initialize(); } - + const applyProviderFromConfig = () => { - const provider = createLLMProvider(); + const provider = createLLMProviderFromConfig(); const agentCore = AgentCore.instance(); agentCore.setLLMProvider(provider); - console.log(`Switched to ${provider.name} provider`); + console.log('LLM provider configured'); }; - // Initial apply applyProviderFromConfig(); - // Subscribe to config changes to hot-swap providers const unsubscribe = configManager.addChangeListener((changedKeys) => { - // Hot-swap on provider change or when relevant provider config changes if ( changedKeys.includes('general.llm_provider') || changedKeys.some(k => k.startsWith('general.openai.')) || - changedKeys.some(k => k.startsWith('general.openai_compatible.')) || - changedKeys.some(k => k.startsWith('general.claude_openrouter.')) || - changedKeys.some(k => k.startsWith('general.gemini.')) || - changedKeys.some(k => k.startsWith('general.claude.')) + changedKeys.some(k => k.startsWith('general.openai_compatible.')) ) { applyProviderFromConfig(); } }); - // Cleanup subscription on unmount return unsubscribe; }; - - // Register the UI clear callback for external components to use + registerClearChatUICallback(clearChatUI); - + const maybeUnsubscribePromise = initializeProvider(); - - // Auto-trigger welcome on first launch (guard against React StrictMode double-invoke only) + (async () => { if (hasShownWelcomeOnceInRuntime) return; hasShownWelcomeOnceInRuntime = true; @@ -214,7 +194,7 @@ const ChatBox: React.FC = ({ isVisible }) => { setMessages([welcomeMessage]); } })(); - // In case initializeProvider returned a cleanup, ensure we call it + return () => { Promise.resolve(maybeUnsubscribePromise).then((cleanup) => { if (typeof cleanup === 'function') cleanup(); @@ -226,17 +206,12 @@ const ChatBox: React.FC = ({ isVisible }) => { const controller = streamProcessor.abortController; if (controller) { controller.abort(); - - // Use AgentCore to clean up the data model and get the user message content + const agentCore = AgentCore.instance(); const userMessageContent = agentCore.abortCurrentRequest(); - - // Remove the last user message and assistant message from UI + setMessages(prev => prev.slice(0, -2)); - - // Restore the user's input (use the content from AgentCore if available) setInputValue(userMessageContent || lastUserMessage); - setIsProcessing(false); } }; @@ -246,101 +221,29 @@ const ChatBox: React.FC = ({ isVisible }) => { clearChatHistoryAndUI(setStatus); }; - const executeToolsFromResponse = async (response: string): Promise => { - try { - const actionableBlocks = extractActionableTools(response); - - if (actionableBlocks.length === 0) { - // No actionable tools to execute, stop the loop - return false; - } - - // Start tool execution phase - setIsExecutingTools(true); - setToolResults(''); - setCurrentToolIndex(0); - - const { setStatus } = useProjectStore.getState(); - - // Execute all tools and get accumulated results - const accumulatedResults = await executeAllTools(actionableBlocks, { - onMessageAdd: handleMessageAdd, - onStatusUpdate: setStatus - }); - - // Store accumulated results - setToolResults(accumulatedResults); - setIsExecutingTools(false); - - // Check if agent is still working on task before sending results to LLM - const agentCore = AgentCore.instance(); - const isStillWorkingOnTask = agentCore.getAgentState().getIsWorkingOnTask(); - - if (isStillWorkingOnTask) { - // Send tool results back to LLM - setStatus('Processing tool results...'); - await sendToolResultsToLLM(accumulatedResults); - } else { - // Agent is no longer working on task, ignore results and return control to user - setStatus('Tool execution completed'); - } - - return true; // Tools were found and executed - - } catch (error) { - console.error('Error executing tools:', error); - setIsExecutingTools(false); - const { setStatus } = useProjectStore.getState(); - setStatus(`Tool execution failed: ${error}`); - return false; // Tool execution failed - } - }; - - const sendToolResultsToLLM = async (toolResultsString: string): Promise => { - // Process tool results through the stream processor - const assistantResponse = await streamProcessor.processStream(toolResultsString, 'TOOL_RESULTS'); - - // Check if the new response contains more tools - const hasMoreTools = await executeToolsFromResponse(assistantResponse); - - // If no more tools were found, set working flag to false - if (!hasMoreTools) { - const agentCore = AgentCore.instance(); - agentCore.getAgentState().setIsWorkingOnTask(false); - } - }; - const handleSend = async () => { if (inputValue.trim() && !isProcessing) { const userMessage = inputValue.trim(); setLastUserMessage(userMessage); setInputValue(''); - // Run message through the filter system const filterResult = await processUserMessage(userMessage); - // Conditionally show the user message bubble if (filterResult.displayUserMessage) { const userMsgObject = createMessage('user', userMessage); handleMessageAdd(userMsgObject); } - // If we have a pseudo assistant response, show it immediately if (filterResult.pseudoAssistantResponse) { const pseudoMessage = createMessage('assistant', filterResult.pseudoAssistantResponse); handleMessageAdd(pseudoMessage); } - // If we shouldn't send anything to LLM, stop here if (!filterResult.sendToLLM || !filterResult.finalMessageForLLM) { return; } - // Set working on task flag when user sends a message - const agentCore = AgentCore.instance(); - agentCore.getAgentState().setIsWorkingOnTask(true); - - // Log system prompt only for first message or first message after clear + // Log system prompt only for first message if (isFirstMessage) { try { const systemPrompt = await SystemPrompts.getSystemPromptWithContext(); @@ -350,20 +253,11 @@ const ChatBox: React.FC = ({ isVisible }) => { } catch (error) { console.error('Failed to log system prompt:', error); } - // Mark that we've logged the system prompt for this conversation setIsFirstMessage(false); } - // Process user input through the stream processor - const assistantResponse = await streamProcessor.processStream(filterResult.finalMessageForLLM, 'USER'); - - // Check if response contains tools to execute - const hasTools = await executeToolsFromResponse(assistantResponse); - - // If no tools were found, set working flag to false and return control to user - if (!hasTools) { - agentCore.getAgentState().setIsWorkingOnTask(false); - } + // Process through stream processor — AgentCore handles the full agentic loop internally + await streamProcessor.processStream(filterResult.finalMessageForLLM, 'USER'); } }; @@ -372,18 +266,15 @@ const ChatBox: React.FC = ({ isVisible }) => { e.preventDefault(); handleSend(); } - // Allow Shift+Enter for new lines (default textarea behavior) }; const handleInputFocus = () => { - // Set a data attribute on the textarea to help global keyboard handler identify it if (textareaRef.current) { textareaRef.current.setAttribute('data-chatbox-input', 'true'); } }; const handleInputBlur = () => { - // Remove the data attribute when losing focus if (textareaRef.current) { textareaRef.current.removeAttribute('data-chatbox-input'); } @@ -403,18 +294,15 @@ const ChatBox: React.FC = ({ isVisible }) => { messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }); }, [messages]); - // Auto-focus input when it becomes visible (when processing and tool execution complete) + // Auto-focus input when processing completes useEffect(() => { - if (!isProcessing && !isExecutingTools && textareaRef.current) { - // Use a small delay to ensure the DOM has updated + if (!isProcessing && textareaRef.current) { setTimeout(() => { textareaRef.current?.focus(); - // Also scroll to bottom when input becomes visible after tool execution - // This ensures proper scroll position after layout changes from showing input box messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }); }, 0); } - }, [isProcessing, isExecutingTools]); + }, [isProcessing]); return (
@@ -463,14 +351,14 @@ const ChatBox: React.FC = ({ isVisible }) => {
- +
{messages.map((message) => ( message.role === 'user' ? ( ) : ( - = ({ isVisible }) => { ))}
- - {!isProcessing && !isExecutingTools && ( + + {!isProcessing && (