Merge pull request #8 from KGAudioLab/fix/2025-08-16-misc
Fix/2025 08 16 misc
This commit is contained in:
Xiaohan-Tian
@@ -18,6 +18,7 @@ DEPLOYMENT.md
|
|||||||
# Editor directories and files
|
# Editor directories and files
|
||||||
!.vscode/extensions.json
|
!.vscode/extensions.json
|
||||||
.idea
|
.idea
|
||||||
|
.claude
|
||||||
.DS_Store
|
.DS_Store
|
||||||
*.suo
|
*.suo
|
||||||
*.ntvs*
|
*.ntvs*
|
||||||
|
|||||||
@@ -14,6 +14,8 @@ K.G.Studio is a lightweight, modern DAW that runs entirely in the browser with *
|
|||||||
|
|
||||||
This project investigates how AI-human collaboration can enhance creative music-making, from intelligent harmony suggestions to automated editing tasks. As an experimental platform, expect frequent changes, evolving features, and occasional instability as we push the boundaries of what's possible in AI-assisted music production.
|
This project investigates how AI-human collaboration can enhance creative music-making, from intelligent harmony suggestions to automated editing tasks. As an experimental platform, expect frequent changes, evolving features, and occasional instability as we push the boundaries of what's possible in AI-assisted music production.
|
||||||
|
|
||||||
|
### Start using the app online: [K.G.Studio (kgaudiolab.github.io/kgstudio)](https://kgaudiolab.github.io/kgstudio)
|
||||||
|
|
||||||
## Demo Videos
|
## Demo Videos
|
||||||
|
|
||||||
<div align="center">
|
<div align="center">
|
||||||
@@ -35,6 +37,30 @@ This project investigates how AI-human collaboration can enhance creative music-
|
|||||||
</table>
|
</table>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
### Setting up K.G.Studio Musician Assistant
|
||||||
|
- [Click here to get a free OpenRouter API Key](https://openrouter.ai/keys) (you may need an OpenRouter account).
|
||||||
|
- In **Settings ⚙️ → General → LLM Provider**, select **OpenAI Compatible**.
|
||||||
|
- In **OpenAI Compatible Server → Key**, paste your key. (Note: on non‑localhost, your key isn’t persisted; you’ll re‑enter it after refresh the page. Keep it safe.)
|
||||||
|
- In **OpenAI Compatible Server → Model**, enter `qwen/qwen3-30b-a3b:free`. (Note: non‑free models may require billing; some model providers may retain your data, check their privacy policies; this project is not affiliated with OpenRouter or any model provider.)
|
||||||
|
- In **OpenAI Compatible Server → Base URL**, enter `https://openrouter.ai/api/v1/chat/completions`.
|
||||||
|
|
||||||
|
(Alternatively, you can use the official OpenAI API, other OpenAI‑compatible services, or your own hosted LLM server. e.g., Ollama, vLLM)
|
||||||
|
|
||||||
|
### Basic DAW operations
|
||||||
|
- Double‑click (or hold Ctrl/Cmd and click) on a track to create a region.
|
||||||
|
- Drag region edges to resize; drag the body to move.
|
||||||
|
- Click the small pencil at a region’s top‑left to open the piano roll.
|
||||||
|
- In the piano roll, double‑click (or Ctrl/Cmd+click) to create a note.
|
||||||
|
- Click to select; Shift+click for multi‑select; drag to box‑select.
|
||||||
|
- Drag note edges to resize; drag the note body to move selected notes.
|
||||||
|
- Use Snapping in the piano roll toolbar (top‑right) to quantize to grid.
|
||||||
|
|
||||||
|
### More details
|
||||||
|
|
||||||
|
You can find the detailed user guide [here](./docs/USER_GUIDE.md).
|
||||||
|
|
||||||
### Highlights
|
### Highlights
|
||||||
- **K.G.Studio Musician Assistant**: Chat with the LLM‑powered K.G.Studio Musician Assistant AI Agent; it can automatically execute tools to make music edits.
|
- **K.G.Studio Musician Assistant**: Chat with the LLM‑powered K.G.Studio Musician Assistant AI Agent; it can automatically execute tools to make music edits.
|
||||||
- **Multiple LLM providers**: OpenAI, Claude (via OpenRouter), Gemini (via OpenRouter), or OpenAI‑compatible (e.g., Ollama, OpenRouter).
|
- **Multiple LLM providers**: OpenAI, Claude (via OpenRouter), Gemini (via OpenRouter), or OpenAI‑compatible (e.g., Ollama, OpenRouter).
|
||||||
@@ -50,8 +76,6 @@ For a deeper technical overview, see `res/overview.md`.
|
|||||||
|
|
||||||
## Getting Started
|
## Getting Started
|
||||||
|
|
||||||
### Start using the app online: [K.G.Studio (kgaudiolab.github.io/kgstudio)](https://kgaudiolab.github.io/kgstudio)
|
|
||||||
|
|
||||||
### Or clone and run locally:
|
### Or clone and run locally:
|
||||||
```bash
|
```bash
|
||||||
# Make sure you have Node.js >= 20.19.3 installed
|
# Make sure you have Node.js >= 20.19.3 installed
|
||||||
@@ -94,16 +118,33 @@ K.G.Studio loads defaults from `./public/config.json` (with an internal fallback
|
|||||||
|
|
||||||
You can find the detailed user guide [here](./docs/USER_GUIDE.md).
|
You can find the detailed user guide [here](./docs/USER_GUIDE.md).
|
||||||
|
|
||||||
1. Start the app and a default project will load. A default “Melody” MIDI track is ensured on load.
|
- Tracks
|
||||||
2. Add tracks, rename and reorder them. Click or drag in the track grid to create and arrange regions.
|
- Add, rename, and reorder tracks from the track info panel.
|
||||||
3. Add a region by double‑clicking or holding Ctrl/Cmd and clicking. Drag the edges to resize a region; drag the center to move it. Use Ctrl/Cmd+C and Ctrl/Cmd+V to copy/paste regions.
|
- Change instrument using the instrument button (piano icon); adjust Solo (S), Mute (M), and Volume.
|
||||||
4. Open a MIDI region to edit notes in the Piano Roll window (click the pencil icon on the region’s top‑left corner, or click the Piano icon on the toolbar).
|
- Delete a track from the track’s settings menu (button to the right of the instrument).
|
||||||
5. Double‑click or hold Ctrl/Cmd and click to create a note. Drag the edges to change its length, or drag the center to move it. Use box selection or hold Shift to multi‑select.
|
|
||||||
6. Quantize note length/position using the Quantize dropdowns at the top‑right of the piano roll.
|
|
||||||
7. Change instruments via the Instrument Selection panel; samples are downloaded on demand.
|
|
||||||
8. Save projects to your browser’s IndexedDB and load by name from the toolbar. Export your project as a MIDI file, or import a MIDI file into your project.
|
|
||||||
|
|
||||||
## Keyboard Shortcuts (defaults)
|
- Regions
|
||||||
|
- Create region: with the Pointer tool, double‑click; or hold Ctrl/Cmd and click. With the Pencil tool, single‑click.
|
||||||
|
- Move/resize: drag the body to move; drag edges to resize.
|
||||||
|
- Open Piano Roll via the small pencil at a region’s top‑left.
|
||||||
|
|
||||||
|
- Piano Roll (MIDI notes)
|
||||||
|
- Tools: Select vs Pencil.
|
||||||
|
- Create notes: double‑click or Ctrl/Cmd+click (Select); single‑click (Pencil).
|
||||||
|
- Select notes: click; Shift+click for multi‑select; drag to box‑select.
|
||||||
|
- Move/resize: drag note body to move selected notes; drag edges to resize.
|
||||||
|
- Close the piano roll with X or ESC.
|
||||||
|
|
||||||
|
- Snapping and Quantize
|
||||||
|
- Set snapping from the NO SNAP menu (top‑right).
|
||||||
|
- Quantize timing with Qua. Pos. (start) and Qua. Len. (length).
|
||||||
|
|
||||||
|
- Playback & Playhead
|
||||||
|
- Back to beginning; Play/Pause from the toolbar.
|
||||||
|
- Set the playhead by clicking bar numbers in the main grid; in Piano Roll, click the header timeline (respects snapping).
|
||||||
|
- Change BPM, time signature, and key signature via the toolbar readouts.
|
||||||
|
|
||||||
|
## Keyboard Shortcuts
|
||||||
|
|
||||||
- Main
|
- Main
|
||||||
- Play/Pause: Space
|
- Play/Pause: Space
|
||||||
@@ -120,11 +161,11 @@ You can find the detailed user guide [here](./docs/USER_GUIDE.md).
|
|||||||
|
|
||||||
## AI Assistant
|
## AI Assistant
|
||||||
|
|
||||||
### Quick Start
|
### Using K.G.Studio Musician Assistant
|
||||||
|
|
||||||
- First, select an LLM provider in **Settings ⚙️ → General → LLM Provider**. If you are using OpenAI (e.g., GPT‑4o), get your API key from [**OpenAI**](https://platform.openai.com/account/api-keys) and paste it in **OpenAI → Key**.
|
- Make sure you have followed the previous section to set up the LLM provider.
|
||||||
- You can find the K.G.Studio Musician Assistant chatbox on the right. If you don't see it, you can click the Chat 🗨️ button on the toolbar.
|
- You can find the K.G.Studio Musician Assistant chatbox on the right. If you don't see it, you can click the Chat 🗨️ button on the toolbar.
|
||||||
- Type your message in the chatbox; press Enter to send, Shift+Enter to insert a new line.
|
- Select the region you want the assistant to work on, type your prompt in the chatbox; press Enter to send, Shift+Enter to insert a new line.
|
||||||
- The agent will automatically process your request and invoke tools to make modifications scoped to the selected region. It may take one or more turns to complete a task.
|
- The agent will automatically process your request and invoke tools to make modifications scoped to the selected region. It may take one or more turns to complete a task.
|
||||||
- Note that the AI could make mistakes, so you should always check the result and make adjustments if necessary. You can also use undo/redo to revert the changes.
|
- Note that the AI could make mistakes, so you should always check the result and make adjustments if necessary. You can also use undo/redo to revert the changes.
|
||||||
- Click the "+" button or `/clear` command to clear the chat history.
|
- Click the "+" button or `/clear` command to clear the chat history.
|
||||||
@@ -153,11 +194,9 @@ OpenRouter is a platform that provides unified access to a wide range of languag
|
|||||||
4. Browse available models on the [**OpenRouter Models Page**](https://openrouter.ai/models). Use the "Prompt Pricing" filter to identify free models.
|
4. Browse available models on the [**OpenRouter Models Page**](https://openrouter.ai/models). Use the "Prompt Pricing" filter to identify free models.
|
||||||
**Note:** Each model provider may have different data retention and privacy policies. Please review these policies before use.
|
**Note:** Each model provider may have different data retention and privacy policies. Please review these policies before use.
|
||||||
5. Enter your chosen model name in **OpenAI Compatible Server → Model**. Recommended model series include:
|
5. Enter your chosen model name in **OpenAI Compatible Server → Model**. Recommended model series include:
|
||||||
- `Google: gemini-2.5-pro`
|
- `Anthropic: claude-4-sonnet` (`anthropic/claude-sonnet-4`: [Link](https://openrouter.ai/anthropic/claude-sonnet-4))
|
||||||
- `Anthropic: claude-4-sonnet`
|
- `Qwen: qwen3-30b-a3b` (FREE MODEL: `qwen/qwen3-30b-a3b:free`: [Link](https://openrouter.ai/qwen/qwen3-30b-a3b:free))
|
||||||
- `DeepSeek: deepseek-r1` (free: `deepseek/deepseek-r1-0528:free`)
|
- `Qwen: qwen3-235b-a22b` (FREE MODEL: `qwen/qwen3-235b-a22b:free`: [Link](https://openrouter.ai/qwen/qwen3-235b-a22b:free))
|
||||||
- `DeepSeek: deepseek-v3` (free: `deepseek/deepseek-chat-v3-0324:free`)
|
|
||||||
- `Qwen: qwen3-235b-a22b` (free: `qwen/qwen3-235b-a22b:free`)
|
|
||||||
6. Input the base URL `https://openrouter.ai/api/v1/chat/completions` **OpenAI Compatible Server → Base URL**.
|
6. Input the base URL `https://openrouter.ai/api/v1/chat/completions` **OpenAI Compatible Server → Base URL**.
|
||||||
|
|
||||||
### About the agent and LLM providers
|
### About the agent and LLM providers
|
||||||
|
|||||||
@@ -0,0 +1,600 @@
|
|||||||
|
# KGStudio: Digital Audio Workstation (DAW) Project Overview
|
||||||
|
|
||||||
|
## Project Introduction
|
||||||
|
|
||||||
|
KGStudio is a light-weighted, modern, web-based Digital Audio Workstation (DAW) built with React, TypeScript, and Zustand. The project aims to provide a professional-grade music production environment in the browser, with features comparable to desktop DAWs like Ableton Live, FL Studio, or Logic Pro.
|
||||||
|
|
||||||
|
## Ultimate Goals
|
||||||
|
|
||||||
|
1. Create a fully-functional DAW that runs in modern web browsers
|
||||||
|
2. Provide a professional-grade UI with intuitive workflows for music production
|
||||||
|
3. Support MIDI and audio recording, editing, and playback
|
||||||
|
4. Implement a plugin system for virtual instruments and effects
|
||||||
|
5. Enable project saving, loading, and export functionality
|
||||||
|
6. Optimize for performance to handle complex projects with many tracks
|
||||||
|
|
||||||
|
## Current Tech Stack
|
||||||
|
|
||||||
|
- **Frontend Framework**: React with TypeScript
|
||||||
|
- **State Management**: Zustand
|
||||||
|
- **Build Tool**: Vite
|
||||||
|
- **UI Components**: Custom components with CSS; icons via React Icons
|
||||||
|
- **Audio Engine**: Tone.js for Web Audio synthesis and playback (real soundfonts via Sampler)
|
||||||
|
- **Data Persistence**: IndexedDB using `idb`, class serialization with `class-transformer`
|
||||||
|
- **AI/Agent**: Configurable LLM provider (OpenAI/Claude/Gemini/compatible) with XML tool execution
|
||||||
|
- **Architecture Pattern**: Core/UI separation with a domain model and component-based UI
|
||||||
|
|
||||||
|
## Project Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
KGStudio/
|
||||||
|
├── docs/ # Project documentation
|
||||||
|
│ ├── USER_GUIDE.md # End-user guide
|
||||||
|
│ └── technical/
|
||||||
|
│ └── overview.md # Technical project overview (this document)
|
||||||
|
├── public/ # Static assets
|
||||||
|
│ ├── apple-touch-icon.png # PWA icon (Apple)
|
||||||
|
│ ├── config.json # Default application configuration
|
||||||
|
│ ├── favicon-96x96.png # Favicon (96x96)
|
||||||
|
│ ├── favicon.ico # Favicon (ICO)
|
||||||
|
│ ├── favicon.svg # Favicon (SVG)
|
||||||
|
│ ├── logo.png # Application logo
|
||||||
|
│ ├── logo-kgaudiolab.png # Alternate brand logo (KGAudioLab)
|
||||||
|
│ ├── site.webmanifest # PWA manifest
|
||||||
|
│ ├── web-app-manifest-192x192.png # PWA icon (192x192)
|
||||||
|
│ ├── web-app-manifest-512x512.png # PWA icon (512x512)
|
||||||
|
│ ├── vite.svg # Vite logo
|
||||||
|
│ ├── chat/ # Chat UI copy and errors
|
||||||
|
│ │ ├── error_no_openai_compatible_base_url.md
|
||||||
|
│ │ ├── error_no_openai_compatible_model.md
|
||||||
|
│ │ ├── error_no_openai_key.md
|
||||||
|
│ │ ├── error_no_selected_region.md
|
||||||
|
│ │ ├── welcome_again.md
|
||||||
|
│ │ ├── welcome_new.md
|
||||||
|
│ │ ├── custom_instructions_gpt-4o.md # Custom instructions template for GPT-4o
|
||||||
|
│ │ └── custom_instructions_qwen3-a3b-30b.md # Custom instructions template for Qwen3-A3B-30B
|
||||||
|
│ ├── demo/ # Demo assets
|
||||||
|
│ │ ├── cover-FXgihfAH2vc.png # Demo cover image
|
||||||
|
│ │ └── cover-vKbWAQRt0r0.png # Demo cover image
|
||||||
|
│ ├── prompts/ # AI system prompts
|
||||||
|
│ │ ├── system.md
|
||||||
|
│ │ ├── system_20250806.md
|
||||||
|
│ │ ├── user_msg_appendix.md
|
||||||
|
│ │ └── README.md # Prompt set overview
|
||||||
|
│ └── resources/ # Application resources
|
||||||
|
│ ├── icon.png
|
||||||
|
│ ├── instrument_bg.png
|
||||||
|
│ ├── instrument_bg_v0.png
|
||||||
|
│ └── instruments/ # Instrument icons (General MIDI)
|
||||||
|
│ ├── bass.png
|
||||||
|
│ ├── brass_ensemble.png
|
||||||
|
│ ├── cello.png
|
||||||
|
│ ├── clarinet.png
|
||||||
|
│ ├── contrabass.png
|
||||||
|
│ ├── drawbar_organ.png
|
||||||
|
│ ├── drums.png
|
||||||
|
│ ├── electric_guitar.png
|
||||||
|
│ ├── electric_piano.png
|
||||||
|
│ ├── flute.png
|
||||||
|
│ ├── french_horn.png
|
||||||
|
│ ├── guitar.png
|
||||||
|
│ ├── harp.png
|
||||||
|
│ ├── oboe.png
|
||||||
|
│ ├── orchestra_percussion_kit.png
|
||||||
|
│ ├── piano.png
|
||||||
|
│ ├── sax.png
|
||||||
|
│ ├── string_ensemble.png
|
||||||
|
│ ├── synth.png
|
||||||
|
│ ├── trombone.png
|
||||||
|
│ ├── trumpet.png
|
||||||
|
│ ├── viola.png
|
||||||
|
│ └── violin.png
|
||||||
|
├── src/
|
||||||
|
│ ├── agent/ # AI agent system
|
||||||
|
│ │ ├── core/ # Agent core components
|
||||||
|
│ │ │ ├── AgentCore.ts # Main agent orchestration
|
||||||
|
│ │ │ ├── AgentState.ts # Agent state management
|
||||||
|
│ │ │ ├── SystemPrompts.ts # System prompts for AI
|
||||||
|
│ │ │ └── XMLToolExecutor.ts # XML tool execution engine
|
||||||
|
│ │ ├── llm/ # LLM integration
|
||||||
|
│ │ │ ├── ClaudeProvider.ts
|
||||||
|
│ │ │ ├── GeminiProvider.ts
|
||||||
|
│ │ │ ├── LLMProvider.ts # Abstract LLM provider interface
|
||||||
|
│ │ │ ├── OpenAIProvider.ts # OpenAI API integration
|
||||||
|
│ │ │ └── StreamingTypes.ts # Streaming response types
|
||||||
|
│ │ └── tools/ # Agent tools
|
||||||
|
│ │ ├── AddNotesTool.ts # Tool for adding notes
|
||||||
|
│ │ ├── AttemptCompletionTool.ts # Mark current task as completed
|
||||||
|
│ │ ├── BaseTool.ts # Base tool class
|
||||||
|
│ │ ├── ReadMusicTool.ts # Tool for reading music
|
||||||
|
│ │ ├── RemoveNotesTool.ts # Tool for removing notes
|
||||||
|
│ │ ├── ThinkTool.ts # Background thinking tool
|
||||||
|
│ │ ├── ThinkingTool.ts # Alternative thought tool
|
||||||
|
│ │ └── index.ts # Tool exports
|
||||||
|
│ ├── App.css # Main application styles
|
||||||
|
│ ├── App.tsx # Main application component
|
||||||
|
│ ├── assets/
|
||||||
|
│ │ └── react.svg # React logo
|
||||||
|
│ ├── components/ # React UI components
|
||||||
|
│ │ ├── ChatBox.tsx # Chatbox component
|
||||||
|
│ │ ├── InstrumentSelection.tsx # Instrument picker
|
||||||
|
│ │ ├── TrackControl.tsx # Track control component
|
||||||
|
│ │ ├── chat/
|
||||||
|
│ │ │ ├── AssistantMessage.tsx
|
||||||
|
│ │ │ ├── UserMessage.tsx
|
||||||
|
│ │ │ └── index.ts
|
||||||
|
│ │ ├── common/ # Common reusable components
|
||||||
|
│ │ │ ├── FileImportModal.tsx
|
||||||
|
│ │ │ ├── KGDropdown.tsx
|
||||||
|
│ │ │ ├── LoadingOverlay.tsx
|
||||||
|
│ │ │ ├── Playhead.tsx
|
||||||
|
│ │ │ ├── icons/
|
||||||
|
│ │ │ │ └── PianoIcon.tsx
|
||||||
|
│ │ │ └── index.ts
|
||||||
|
│ │ ├── interfaces.ts # Shared interfaces
|
||||||
|
│ │ ├── MainContent.tsx # Main track display area
|
||||||
|
│ │ ├── piano-roll/ # Piano roll related components
|
||||||
|
│ │ │ ├── PianoGrid.tsx
|
||||||
|
│ │ │ ├── PianoGridHeader.tsx
|
||||||
|
│ │ │ ├── PianoKeys.tsx
|
||||||
|
│ │ │ ├── PianoNote.tsx
|
||||||
|
│ │ │ ├── PianoRoll.tsx
|
||||||
|
│ │ │ ├── PianoRollContent.tsx
|
||||||
|
│ │ │ ├── PianoRollHeader.tsx
|
||||||
|
│ │ │ ├── PianoRollToolbar.tsx
|
||||||
|
│ │ │ └── SelectionBox.tsx
|
||||||
|
│ │ ├── settings/
|
||||||
|
│ │ │ ├── SettingsPanel.tsx
|
||||||
|
│ │ │ ├── SettingsSidebar.tsx
|
||||||
|
│ │ │ ├── index.ts
|
||||||
|
│ │ │ └── sections/
|
||||||
|
│ │ │ ├── BehaviorSettings.tsx
|
||||||
|
│ │ │ ├── GeneralSettings.tsx
|
||||||
|
│ │ │ └── TemplatesSettings.tsx
|
||||||
|
│ │ ├── StatusBar.tsx # Status bar component
|
||||||
|
│ │ ├── Toolbar.tsx # Top toolbar component
|
||||||
|
│ │ └── track/
|
||||||
|
│ │ ├── RegionItem.tsx
|
||||||
|
│ │ ├── TrackGridItem.tsx
|
||||||
|
│ │ ├── TrackGridPanel.tsx
|
||||||
|
│ │ ├── TrackInfoItem.tsx
|
||||||
|
│ │ └── TrackInfoPanel.tsx
|
||||||
|
│ ├── constants/ # Application constants
|
||||||
|
│ │ ├── coreConstants.ts # Core application constants (DB, audio, etc.)
|
||||||
|
│ │ ├── generalMidiConstants.ts # General MIDI mapping/constants
|
||||||
|
│ │ ├── index.ts # Constants re-export
|
||||||
|
│ │ ├── midiConstants.ts # MIDI message constants
|
||||||
|
│ │ └── uiConstants.ts # UI-related constants
|
||||||
|
│ ├── core/ # Core application logic
|
||||||
|
│ │ ├── KGCore.ts # Main application singleton
|
||||||
|
│ │ ├── KGDebugger.ts # Debug utilities and testing tools
|
||||||
|
│ │ ├── KGProject.ts # Project model
|
||||||
|
│ │ ├── audio-interface/ # Advanced audio engine integration
|
||||||
|
│ │ │ ├── KGAudioBus.ts # Individual track audio bus
|
||||||
|
│ │ │ ├── KGAudioInterface.ts # Audio engine coordinator
|
||||||
|
│ │ │ ├── KGToneBuffersPool.ts # Soundfont buffers pool
|
||||||
|
│ │ │ └── KGToneSamplerFactory.ts # Sampler factory
|
||||||
|
│ │ ├── commands/ # Command pattern implementation for undo/redo
|
||||||
|
│ │ │ ├── KGCommand.ts # Base command class
|
||||||
|
│ │ │ ├── KGCommandHistory.ts # Command history manager
|
||||||
|
│ │ │ ├── index.ts # Command exports
|
||||||
|
│ │ │ ├── note/
|
||||||
|
│ │ │ │ ├── CreateNoteCommand.ts
|
||||||
|
│ │ │ │ ├── CreateNotesCommand.ts
|
||||||
|
│ │ │ │ ├── DeleteNotesCommand.ts
|
||||||
|
│ │ │ │ ├── MoveNotesCommand.ts
|
||||||
|
│ │ │ │ ├── PasteNotesCommand.ts
|
||||||
|
│ │ │ │ └── ResizeNotesCommand.ts
|
||||||
|
│ │ │ ├── project/
|
||||||
|
│ │ │ │ └── ChangeProjectPropertyCommand.ts
|
||||||
|
│ │ │ ├── track/
|
||||||
|
│ │ │ │ ├── AddTrackCommand.ts # Track command: add track
|
||||||
|
│ │ │ │ ├── RemoveTrackCommand.ts # Track command: remove track
|
||||||
|
│ │ │ │ ├── ReorderTracksCommand.ts # Track command: reorder tracks
|
||||||
|
│ │ │ │ └── UpdateTrackCommand.ts # Track command: update track
|
||||||
|
│ │ │ └── region/
|
||||||
|
│ │ │ ├── CreateRegionCommand.ts
|
||||||
|
│ │ │ ├── DeleteRegionCommand.ts
|
||||||
|
│ │ │ ├── MoveRegionCommand.ts
|
||||||
|
│ │ │ ├── PasteRegionsCommand.ts
|
||||||
|
│ │ │ ├── ResizeRegionCommand.ts
|
||||||
|
│ │ │ └── UpdateRegionCommand.ts
|
||||||
|
│ │ ├── config/
|
||||||
|
│ │ │ ├── ConfigManager.ts # JSON config loading/persistence
|
||||||
|
│ │ │ └── index.ts
|
||||||
|
│ │ ├── io/
|
||||||
|
│ │ │ └── KGStorage.ts # IndexedDB storage layer
|
||||||
|
│ │ ├── midi/
|
||||||
|
│ │ │ └── KGMidiNote.ts
|
||||||
|
│ │ ├── project-upgrader/
|
||||||
|
│ │ │ ├── KGProjectUpgrader.ts
|
||||||
|
│ │ │ └── upgradeToV1.ts
|
||||||
|
│ │ ├── region/
|
||||||
|
│ │ │ ├── KGMidiRegion.ts
|
||||||
|
│ │ │ └── KGRegion.ts
|
||||||
|
│ │ ├── state/
|
||||||
|
│ │ │ ├── KGMainContentState.ts
|
||||||
|
│ │ │ └── KGPianoRollState.ts
|
||||||
|
│ │ └── track/
|
||||||
|
│ │ ├── KGMidiTrack.ts
|
||||||
|
│ │ └── KGTrack.ts
|
||||||
|
│ ├── hooks/
|
||||||
|
│ │ ├── useConfig.ts
|
||||||
|
│ │ ├── useGlobalKeyboardHandler.ts
|
||||||
|
│ │ ├── useNoteOperations.ts
|
||||||
|
│ │ ├── useNoteSelection.ts
|
||||||
|
│ │ └── useRegionOperations.ts
|
||||||
|
│ ├── index.css # Global styles
|
||||||
|
│ ├── main.tsx # Application entry point
|
||||||
|
│ ├── mock/
|
||||||
|
│ │ └── mockChat.ts # Mock chat data for testing
|
||||||
|
│ ├── stores/
|
||||||
|
│ │ └── projectStore.ts # Project state management (Zustand)
|
||||||
|
│ ├── types/
|
||||||
|
│ │ └── projectTypes.ts # TypeScript types
|
||||||
|
│ └── util/ # Utility functions
|
||||||
|
│ ├── abcNotationUtil.ts
|
||||||
|
│ ├── chatUtil.ts
|
||||||
|
│ ├── copyPasteUtil.ts
|
||||||
|
│ ├── mathUtil.ts
|
||||||
|
│ ├── messageFilter/
|
||||||
|
│ │ └── UserMessageFilter.ts
|
||||||
|
│ ├── midiUtil.ts
|
||||||
|
│ ├── miscUtil.ts
|
||||||
|
│ ├── osUtil.ts
|
||||||
|
│ ├── regionDeleteUtil.ts
|
||||||
|
│ ├── saveUtil.ts
|
||||||
|
│ ├── timeUtil.ts
|
||||||
|
│ └── xmlUtil.ts
|
||||||
|
│
|
||||||
|
│ └── vite-env.d.ts # Vite environment types
|
||||||
|
├── CLAUDE.md # Claude-specific documentation
|
||||||
|
├── LICENSE # Project license
|
||||||
|
├── eslint.config.js # ESLint configuration
|
||||||
|
├── index.html # HTML entry point
|
||||||
|
├── package-lock.json # NPM lock file
|
||||||
|
├── package.json # NPM dependencies and scripts
|
||||||
|
├── README.md # Project readme
|
||||||
|
├── tsconfig.app.json # App-specific TypeScript config
|
||||||
|
├── tsconfig.json # TypeScript configuration
|
||||||
|
├── tsconfig.node.json # Node-specific TypeScript config
|
||||||
|
├── vite.config.ts # Vite build configuration
|
||||||
|
└── .gitignore # Git ignore rules
|
||||||
|
```
|
||||||
|
|
||||||
|
## Current Implementation
|
||||||
|
|
||||||
|
### Core Architecture
|
||||||
|
|
||||||
|
The application follows a core/UI separation pattern:
|
||||||
|
|
||||||
|
1. **KGCore** (Singleton): The main application class that manages the audio engine, project state, command history, and provides a global access point.
|
||||||
|
2. **Command System**: Complete undo/redo implementation using the Command Pattern:
|
||||||
|
- **KGCommand**: Abstract base class for all undoable operations
|
||||||
|
- **KGCommandHistory**: Manages command history with undo/redo stack and memory limits
|
||||||
|
- **Track Commands**: Add, remove, reorder, and update tracks with full undo support
|
||||||
|
- **Region Commands**: Create, delete, resize, move, paste, and update regions with undo support
|
||||||
|
- **Note Commands**: Create, delete, resize, move, and paste notes with undo support
|
||||||
|
- **Project Commands**: Change project properties (name, BPM, time signature) with selective undo
|
||||||
|
3. **ConfigManager** (Singleton): Manages application configuration including hotkeys, general settings (LLM provider, API keys, soundfont base URL), behavior (e.g., chatbox default open), and templates. Loads defaults from `/public/config.json` (with robust fallback) and persists user customizations via KGStorage.
|
||||||
|
4. **KGStorage** (Singleton): Generic storage system providing unified access to IndexedDB for projects, configuration, and other data. Replaces individual storage implementations with a centralized, reusable storage layer.
|
||||||
|
5. **KGAudioInterface** (Singleton): High-level audio engine coordinator that manages track audio buses and orchestrates playback.
|
||||||
|
6. **KGAudioBus**: Individual track audio processing unit with realistic instrument samples, volume, mute, solo, and effect chain support.
|
||||||
|
7. **KGToneSamplerFactory** (Singleton): Factory for creating Tone.js samplers loaded with high-quality soundfont samples.
|
||||||
|
8. **KGToneBuffersPool** (Singleton): Efficient buffer management system that loads and caches soundfont audio data from remote CDNs.
|
||||||
|
9. **KGProject**: Represents a music project with properties like name, BPM, time signature, and tracks.
|
||||||
|
10. **KGTrack/KGMidiTrack**: Represents a track in the project with instrument support for MIDI tracks.
|
||||||
|
11. **Project Upgrader**: On load/import, `upgradeProjectToLatest` migrates legacy projects to the latest `KGProject.CURRENT_PROJECT_STRUCTURE_VERSION` (e.g., instrument mapping in V1).
|
||||||
|
11. **KGRegion/KGMidiRegion**: Represents a region in a track (a MIDI clip, audio clip, etc.).
|
||||||
|
12. **KGMidiNote**: Represents a MIDI note with properties like pitch, velocity, start and end beats.
|
||||||
|
|
||||||
|
### Component Architecture
|
||||||
|
|
||||||
|
The UI follows a hierarchical component-based architecture:
|
||||||
|
|
||||||
|
1. **App**: Main container component that orchestrates the overall application layout
|
||||||
|
2. **Toolbar**: Handles top toolbar functionality including transport controls and project name
|
||||||
|
3. **MainContent**: Coordinates between track info and grid panels, manages data flow
|
||||||
|
- **TrackInfoPanel**: Container for track information panels
|
||||||
|
- **TrackInfoItem**: Individual track information panel with controls
|
||||||
|
- **TrackGridPanel**: Container for track grid areas
|
||||||
|
- **TrackGridItem**: Individual track grid with regions
|
||||||
|
- **RegionItem**: Individual region within a track
|
||||||
|
4. **TrackControl**: Provides controls for adding and managing tracks
|
||||||
|
5. **StatusBar**: Displays application status information
|
||||||
|
6. **PianoRoll**: Modal component for MIDI note editing
|
||||||
|
- **PianoRollHeader**: Header component with title and close button
|
||||||
|
- **PianoRollToolbar**: Toolbar with editing tools and quantization options
|
||||||
|
- **PianoRollContent**: Main content area that orchestrates piano roll components
|
||||||
|
- **PianoGridHeader**: Bar numbers display at the top of the grid
|
||||||
|
- **PianoKeys**: Piano keyboard visualization on the left side
|
||||||
|
- **PianoGrid**: Main grid area for note editing
|
||||||
|
- **PianoNote**: Individual MIDI note component
|
||||||
|
- **SelectionBox**: Box selection UI for selecting multiple notes
|
||||||
|
7. **Common Components**: Reusable UI components
|
||||||
|
- **KGDropdown**: Reusable dropdown component for consistent dropdown behavior
|
||||||
|
|
||||||
|
### Settings System
|
||||||
|
|
||||||
|
- Dedicated settings UI with sections: `General`, `Behavior`, `Templates`.
|
||||||
|
- Uses `ConfigManager` and `useConfig` hook for auto-load and debounced save.
|
||||||
|
- General: LLM provider/model/keys, OpenAI-compatible base URL, soundfont base URL.
|
||||||
|
- Behavior: chatbox default open at startup.
|
||||||
|
- Templates: custom instructions for the agent.
|
||||||
|
|
||||||
|
### AI Agent & Chat Integration
|
||||||
|
|
||||||
|
- `ChatBox` integrates an `AgentCore` with pluggable LLM providers (OpenAI, Claude, Gemini, OpenAI-compatible) selected via settings.
|
||||||
|
- User messages pass through `UserMessageFilter` supporting slash-commands: `/clear`, `/welcome` and context validation (e.g., require a selected region).
|
||||||
|
- Agent responses can include XML tool calls executed by `XMLToolExecutor` with tools such as `AddNotesTool`, `RemoveNotesTool`, `ReadMusicTool`, `ThinkTool`.
|
||||||
|
- System and user prompt appendix are loaded from `/public/prompts` for richer context.
|
||||||
|
|
||||||
|
### Instrument & Soundfont System
|
||||||
|
|
||||||
|
- Instrument selection panel with groups and previews using `FLUIDR3_INSTRUMENT_MAP` and General MIDI groupings.
|
||||||
|
- Realistic playback via Tone.Sampler created by `KGToneSamplerFactory` and buffers from `KGToneBuffersPool` (downloaded from a configurable CDN).
|
||||||
|
- Global loading overlay shows while instrument buffers are loading; auto-hides and warns if loading takes too long.
|
||||||
|
|
||||||
|
### Keyboard Shortcuts
|
||||||
|
|
||||||
|
- Shortcuts are configurable via `config.json` and `ConfigManager`.
|
||||||
|
- Global: play/pause, undo/redo, copy/cut/paste, save, hold-to-create-region.
|
||||||
|
- Piano roll: tool switching (select/pencil), hold-to-create-note, snapping presets and quantize position/length presets.
|
||||||
|
|
||||||
|
This structure provides clear separation of concerns and improves maintainability.
|
||||||
|
|
||||||
|
### Custom Hooks
|
||||||
|
|
||||||
|
The application uses custom hooks to extract and reuse complex logic:
|
||||||
|
|
||||||
|
1. **useNoteOperations**: Manages note creation, resizing, and dragging operations
|
||||||
|
2. **useNoteSelection**: Manages note selection, including individual selection and box selection
|
||||||
|
3. **useGlobalKeyboardHandler**: Handles global keyboard shortcuts for copy/paste functionality across the application
|
||||||
|
|
||||||
|
### State Management
|
||||||
|
|
||||||
|
Zustand is used for state management with a primary store, complemented by singleton state classes:
|
||||||
|
|
||||||
|
1. **projectStore**: Manages the project state including project name, tracks, BPM, time signature, and provides actions for modifying the project.
|
||||||
|
2. **KGPianoRollState**: Singleton state management for piano roll settings including active tool, snapping options, and quantization preferences.
|
||||||
|
|
||||||
|
### Current Features
|
||||||
|
|
||||||
|
1. **Project Management**:
|
||||||
|
- Create and load projects
|
||||||
|
- Set project properties (name, BPM, time signature)
|
||||||
|
- **Project persistence**: Save and load projects using IndexedDB with automatic serialization
|
||||||
|
- **File operations**: New, Load, Save, and Export buttons in toolbar with confirmation dialogs
|
||||||
|
- **Automatic data model serialization**: All project data (tracks, regions, notes) automatically preserved
|
||||||
|
|
||||||
|
2. **Track Management**:
|
||||||
|
- Add tracks with realistic instrument selection (Piano, Guitar, Bass, Drums)
|
||||||
|
- Rename tracks
|
||||||
|
- Reorder tracks via drag and drop
|
||||||
|
- Advanced track controls (volume, solo, mute) with real-time audio processing
|
||||||
|
- **Modular Audio Bus Architecture**: Each track gets a dedicated KGAudioBus with realistic instrument samples
|
||||||
|
- **Async Track Creation**: Tracks load high-quality soundfont samples on creation
|
||||||
|
- **Instrument Switching**: Change track instruments with seamless audio transitions
|
||||||
|
|
||||||
|
3. **Region Management**:
|
||||||
|
- Create regions by double-clicking on track grid
|
||||||
|
- Regions display with headers and content areas
|
||||||
|
- Regions are tied to the data model (KGMidiRegion)
|
||||||
|
- Regions maintain proper positioning when tracks are reordered
|
||||||
|
- Resize regions from both start and end edges with bar snapping
|
||||||
|
- **Expand from beginning**: Left edge resize properly adjusts note positions to maintain absolute timing
|
||||||
|
- Move regions horizontally within tracks and vertically between tracks via drag and drop
|
||||||
|
- Visual feedback during resize and drag operations (cursor changes, animation effects)
|
||||||
|
- **Canvas-based note visualization**: Real-time display of MIDI notes as white horizontal lines within regions
|
||||||
|
- **Dynamic pitch centering**: Automatically centers display around note content or C4 as fallback
|
||||||
|
- **Adaptive pitch spacing**: Compresses note spacing when range is large to ensure all notes are visible
|
||||||
|
|
||||||
|
4. **Piano Roll**:
|
||||||
|
- Draggable/resizable panel
|
||||||
|
- Piano keyboard visualization
|
||||||
|
- Grid visualization for note editing
|
||||||
|
- Auto-scroll to middle C
|
||||||
|
- Toolbar with editing tools (pointer, pencil)
|
||||||
|
- Snapping options with triplet support for note positioning during drag operations
|
||||||
|
- Quantization options for note position and length with triplet support
|
||||||
|
- Keyboard shortcut (ESC) to close the piano roll
|
||||||
|
- Rename dialog prevention during drag operations
|
||||||
|
- Note creation with double-click
|
||||||
|
- Note resizing from both edges with minimum length constraint
|
||||||
|
- Note dragging with horizontal snapping and vertical movement
|
||||||
|
- Multi-note operations: resize and drag deltas applied to all selected notes
|
||||||
|
- Visual feedback during resize and drag operations
|
||||||
|
- Note selection with click and shift+click for multi-selection
|
||||||
|
- Box selection with shift+drag toggle behavior for multi-selection
|
||||||
|
- Quantize position: snap selected notes to nearest beat grid based on time signature
|
||||||
|
- Quantize length: adjust note durations with smart extension for short notes
|
||||||
|
- Integration with core selection system and singleton state management
|
||||||
|
- **Note Preview with Real Instruments**: Immediate audio feedback when creating notes using authentic instrument samples
|
||||||
|
|
||||||
|
5. **Configuration & Storage Management**:
|
||||||
|
- **JSON-based configuration**: Default settings loaded from `/public/config.json` including hotkeys and general preferences
|
||||||
|
- **User customization persistence**: ConfigManager maintains user overrides while preserving defaults
|
||||||
|
- **Generic storage layer**: KGStorage provides unified IndexedDB access for all data types (projects, config, etc.)
|
||||||
|
- **Centralized constants**: All database and storage constants managed in `coreConstants.ts`
|
||||||
|
- **Type-safe configuration**: Full TypeScript interfaces for configuration structure validation
|
||||||
|
|
||||||
|
6. **Data Persistence**:
|
||||||
|
- **Automatic serialization/deserialization**: Uses class-transformer for seamless data conversion
|
||||||
|
- **Class instance preservation**: All objects maintain their methods and inheritance after save/load
|
||||||
|
- **Browser-based storage**: Projects stored locally using IndexedDB for offline capability
|
||||||
|
- **Type-safe data handling**: Full TypeScript support with proper class instantiation
|
||||||
|
|
||||||
|
7. **Advanced Audio Engine & Realistic Instruments**:
|
||||||
|
- **Professional Soundfont Integration**: High-quality instrument samples from FluidR3_GM soundfonts via remote CDN loading
|
||||||
|
- **Realistic Instrument Library**: Authentic piano, guitar, bass, and drum sounds replacing synthetic oscillators
|
||||||
|
- **Modular Audio Bus System**: Each track operates through a dedicated KGAudioBus with complete audio processing chain
|
||||||
|
- **Intelligent Buffer Management**: KGToneBuffersPool efficiently loads and caches audio samples (A0-C8 range) on-demand
|
||||||
|
- **Async Audio Factory Pattern**: KGToneSamplerFactory creates fully-loaded samplers with proper error handling
|
||||||
|
- **Real-time Audio Playback**: Full project playback with BPM-accurate timing using realistic instrument samples
|
||||||
|
- **Automatic Audio Context Management**: Browser autoplay policy compliance with graceful fallback
|
||||||
|
- **Advanced Track Controls**: Individual track volume, mute, solo with sophisticated audio routing
|
||||||
|
- **Note Preview with Real Instruments**: Immediate audio feedback using actual instrument samples with proper durations
|
||||||
|
- **Future-Ready Architecture**: Prepared for audio effects, filters, and advanced routing capabilities
|
||||||
|
|
||||||
|
8. **Playhead & Transport Controls**:
|
||||||
|
- **Visual playhead indicator**: Blue-green vertical line with triangular marker showing current position
|
||||||
|
- **Interactive timeline navigation**: Click on bar numbers to jump playhead to nearest bar start
|
||||||
|
- **Dual-context rendering**: Playhead appears in both main track view and piano roll with proper positioning
|
||||||
|
- **Play/pause functionality**: Transport controls with state management and visual feedback
|
||||||
|
- **Timer-based playback**: BPM-accurate playhead movement during playback
|
||||||
|
- **Back to beginning**: Quick reset button to return playhead to project start
|
||||||
|
- **Automatic reset**: Playhead resets to position 0 when loading projects
|
||||||
|
- **Reactive state management**: Real-time UI updates synchronized between core engine and interface
|
||||||
|
|
||||||
|
9. **Comprehensive Undo/Redo System**:
|
||||||
|
- **Command Pattern Architecture**: All operations implemented as undoable commands
|
||||||
|
- **Full Operation Coverage**: Undo/redo support for tracks, regions, notes, and project properties
|
||||||
|
- **Selective Property Tracking**: Only modified properties are tracked for efficient undo operations
|
||||||
|
- **Keyboard Shortcuts**: Ctrl/Cmd+Z for undo, Ctrl/Cmd+Y for redo with visual feedback
|
||||||
|
- **UI Integration**: Real-time undo/redo state displayed in toolbar with operation descriptions
|
||||||
|
- **Memory Management**: Configurable command history limits to prevent memory leaks
|
||||||
|
- **State Consistency**: Proper UI synchronization after undo/redo operations
|
||||||
|
- **Multi-Selection Support**: Batch operations on multiple selected items with single undo entry
|
||||||
|
|
||||||
|
10. **Copy & Paste System**:
|
||||||
|
- **Keyboard shortcuts**: Ctrl/Cmd+C and Ctrl/Cmd+V for copy/paste operations
|
||||||
|
- **Toolbar buttons**: Dedicated Copy and Paste buttons in the main toolbar
|
||||||
|
- **Context-aware pasting**: Intelligent paste behavior based on current context (regions vs. notes)
|
||||||
|
- **Region copy/paste**: Copy and paste MIDI regions between tracks at playhead position with undo support
|
||||||
|
- **Note copy/paste**: Copy and paste MIDI notes within piano roll editor with undo support
|
||||||
|
- **Reusable utilities**: Centralized copy/paste logic in `copyPasteUtil.ts` for consistent behavior
|
||||||
|
- **Status feedback**: Real-time status messages showing copy/paste operation results
|
||||||
|
|
||||||
|
11. **UI/UX**:
|
||||||
|
- Responsive layout
|
||||||
|
- Status bar with system messages
|
||||||
|
- Transport display (position, BPM, time signature)
|
||||||
|
- Component-based architecture for better maintainability
|
||||||
|
- Interactive cursors for different operations (grab/grabbing for moving, resize cursors for edges)
|
||||||
|
- Reusable UI components for consistent behavior (KGDropdown)
|
||||||
|
- Proper z-index management for overlapping UI elements
|
||||||
|
- Custom hooks for complex UI logic
|
||||||
|
|
||||||
|
12. **Settings Panel**:
|
||||||
|
- General, Behavior, Templates sections with persistent config and debounced saves
|
||||||
|
- LLM provider switching without reload
|
||||||
|
|
||||||
|
13. **AI Assistant**:
|
||||||
|
- Chat with slash-commands (`/clear`, `/welcome`)
|
||||||
|
- Region-aware messaging; tool-execution pipeline for music edits
|
||||||
|
|
||||||
|
14. **Instrument Library**:
|
||||||
|
- General MIDI-based instrument browsing with icons and real soundfonts
|
||||||
|
- Instant note preview using the selected instrument
|
||||||
|
|
||||||
|
15. **Project Upgrader**:
|
||||||
|
- Automatic migration of legacy projects to the latest structure version
|
||||||
|
|
||||||
|
16. **Global Loading Overlay**:
|
||||||
|
- Shows while soundfonts download; safety timeout with user guidance if loading stalls
|
||||||
|
|
||||||
|
## Features To Be Implemented
|
||||||
|
|
||||||
|
1. **Audio Engine Enhancement**:
|
||||||
|
- Audio recording capabilities
|
||||||
|
- MIDI input/output support
|
||||||
|
- Real-time audio processing and effects
|
||||||
|
- Audio effects and plugin support
|
||||||
|
|
||||||
|
2. **Advanced Playback Features**:
|
||||||
|
- Loop regions and loop playback
|
||||||
|
- Tempo automation and tempo changes during playback
|
||||||
|
- Audio-visual waveform synchronization
|
||||||
|
- Latency compensation and high-precision timing
|
||||||
|
|
||||||
|
3. **Advanced Region Features**:
|
||||||
|
- Region splitting and merging
|
||||||
|
- Audio waveform visualization for audio regions
|
||||||
|
|
||||||
|
4. **Advanced Track Features**:
|
||||||
|
- Track types (MIDI, audio, instrument, etc.)
|
||||||
|
- Track effects and processing
|
||||||
|
- Track automation
|
||||||
|
|
||||||
|
5. **MIDI Editing**:
|
||||||
|
- Velocity editing
|
||||||
|
- MIDI CC automation
|
||||||
|
|
||||||
|
6. **Audio Editing**:
|
||||||
|
- Waveform visualization
|
||||||
|
- Audio clip editing
|
||||||
|
- Audio effects
|
||||||
|
|
||||||
|
7. **Advanced Project Management**:
|
||||||
|
- Export to audio formats
|
||||||
|
- Project templates
|
||||||
|
|
||||||
|
8. **Performance Optimizations**:
|
||||||
|
- Virtualized track rendering
|
||||||
|
- Audio buffer management
|
||||||
|
- Worker threads for processing
|
||||||
|
|
||||||
|
9. **Plugin System**:
|
||||||
|
- Virtual instrument support
|
||||||
|
- Effect plugin support
|
||||||
|
- Plugin browser and management
|
||||||
|
|
||||||
|
## Development Guidelines
|
||||||
|
|
||||||
|
1. **Architecture**:
|
||||||
|
- Maintain separation between core logic and UI
|
||||||
|
- Use TypeScript interfaces for all models
|
||||||
|
- Follow the singleton pattern for global services
|
||||||
|
- Implement Command Pattern for all undoable operations
|
||||||
|
- Structure UI as modular, self-contained components
|
||||||
|
- Follow component hierarchy with clear responsibilities
|
||||||
|
- Extract complex logic to custom hooks
|
||||||
|
|
||||||
|
2. **State Management**:
|
||||||
|
- Keep UI state in Zustand stores
|
||||||
|
- Sync UI state with core models
|
||||||
|
- Use command pattern for all data modifications
|
||||||
|
- Execute commands through KGCore for undo/redo support
|
||||||
|
- Keep component state local when appropriate
|
||||||
|
- Use refs for tracking async state updates
|
||||||
|
|
||||||
|
3. **UI Development**:
|
||||||
|
- Follow existing CSS patterns and variables
|
||||||
|
- Maintain responsive design
|
||||||
|
- Optimize for performance with React best practices
|
||||||
|
- Extract reusable components to improve maintainability
|
||||||
|
- Keep related functionality together in the same component
|
||||||
|
- Ensure proper z-index management for overlapping UI elements
|
||||||
|
- Organize components by functionality (piano, track, etc.)
|
||||||
|
|
||||||
|
4. **Component Communication**:
|
||||||
|
- Parent components provide data via props
|
||||||
|
- Child components notify parents via callbacks
|
||||||
|
- UI components handle their own interactions
|
||||||
|
- Parent components coordinate data model updates
|
||||||
|
- Use custom hooks to share logic between components
|
||||||
|
|
||||||
|
5. **Configuration and Constants**:
|
||||||
|
- Use centralized constants in the constants folder (`coreConstants.ts` for core, `uiConstants.ts` for UI)
|
||||||
|
- Default application settings defined in `/public/config.json`
|
||||||
|
- ConfigManager handles user customizations with persistent storage
|
||||||
|
- KGStorage provides generic access to IndexedDB for all storage needs
|
||||||
|
- Debug mode flags control console logging for different components
|
||||||
|
- Use refs for tracking async state updates
|
||||||
|
- Verify model updates after they occur
|
||||||
|
|
||||||
|
6. **Testing**:
|
||||||
|
- Write unit tests for core logic
|
||||||
|
- Write integration tests for UI components
|
||||||
|
- Test audio processing with specialized audio testing tools
|
||||||
|
|
||||||
|
## Current Limitations
|
||||||
|
|
||||||
|
1. Limited keyboard shortcuts
|
||||||
|
2. Limited note editing capabilities (no velocity editing)
|
||||||
|
3. Export functionality UI only (no actual export implementation)
|
||||||
|
4. No audio recording capabilities yet
|
||||||
|
5. Limited audio effects and processing
|
||||||
|
6. No project browser UI (projects load/save by typing names)
|
||||||
|
|
||||||
|
This document serves as a high-level overview of the KGStudio project, its architecture, current implementation, and future development plans. It should be updated as the project evolves to reflect the current state and goals.
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
# From: {role}
|
||||||
|
|
||||||
|
{timestamp}
|
||||||
|
|
||||||
|
{content}
|
||||||
|
|
||||||
|
---
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
## Help
|
||||||
|
|
||||||
|
Welcome to **K.G.Studio Musician Assistant**—your AI-powered agent for music composition and arrangement. Harnessing advanced language models, I am here to help you create, arrange, and refine your musical ideas with ease and intelligence.
|
||||||
|
|
||||||
|
To get started, you will need to configure an LLM (Large Language Model) Provider in the application settings. Please follow the instructions below to ensure proper setup:
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Configuring Your LLM Provider
|
||||||
|
|
||||||
|
Navigate to **Settings ⚙️ → General → LLM Provider**. Depending on your chosen provider, you will need to supply the appropriate API Key and, if applicable, a custom base URL (for non-official OpenAI-compatible services such as Ollama, OpenRouter, etc.).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Using the OpenAI GPT Model Series
|
||||||
|
|
||||||
|
1. Obtain an OpenAI API Key from [**OpenAI**](https://platform.openai.com/account/api-keys). You may need to create an account and add a payment method to generate an API Key.
|
||||||
|
2. In **Settings ⚙️ → General → LLM Provider**, select **OpenAI** as your provider.
|
||||||
|
3. Enter your API Key in **OpenAI → Key**.
|
||||||
|
4. Select your preferred model from the **OpenAI → Model** dropdown. For optimal performance, we recommend `gpt-5` or `gpt-4o`.
|
||||||
|
5. Optionally, choose whether to enable Flex Mode in **OpenAI → Flex Mode**. Flex Mode offers discounted pricing, but may result in slower response times.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Using OpenRouter
|
||||||
|
|
||||||
|
OpenRouter is a platform that provides unified access to a wide range of language models—including free options—from various providers. This makes it easy to experiment and find the model that best suits your needs.
|
||||||
|
|
||||||
|
1. Obtain an API Key from [**OpenRouter**](https://openrouter.ai/keys). Registration is required; for paid models, a payment method may also be necessary.
|
||||||
|
2. In **Settings ⚙️ → General → LLM Provider**, select **OpenAI Compatible** as your provider.
|
||||||
|
3. Enter your API Key in **OpenAI Compatible Server → Key**.
|
||||||
|
4. Browse available models on the [**OpenRouter Models Page**](https://openrouter.ai/models). Use the "Prompt Pricing" filter to identify free models.
|
||||||
|
**Note:** Each model provider may have different data retention and privacy policies. Please review these policies before use.
|
||||||
|
5. Enter your chosen model name in **OpenAI Compatible Server → Model**. Recommended model series include:
|
||||||
|
- `Anthropic: claude-4-sonnet` (`anthropic/claude-sonnet-4`: [Link](https://openrouter.ai/anthropic/claude-sonnet-4))
|
||||||
|
- `Qwen: qwen3-30b-a3b` (FREE MODEL: `qwen/qwen3-30b-a3b:free`: [Link](https://openrouter.ai/qwen/qwen3-30b-a3b:free))
|
||||||
|
- `Qwen: qwen3-235b-a22b` (FREE MODEL: `qwen/qwen3-235b-a22b:free`: [Link](https://openrouter.ai/qwen/qwen3-235b-a22b:free))
|
||||||
|
6. Input the base URL `https://openrouter.ai/api/v1/chat/completions` of the OpenAI Compatible Server in **OpenAI Compatible Server → Base URL**.
|
||||||
|
|
||||||
|
### Basic DAW Operations
|
||||||
|
|
||||||
|
- Tracks
|
||||||
|
- Add, rename, and reorder tracks from the track info panel.
|
||||||
|
- Change instrument using the instrument button (piano icon); adjust Solo (S), Mute (M), and Volume.
|
||||||
|
- Delete a track from the track’s settings menu (button to the right of the instrument).
|
||||||
|
|
||||||
|
- Regions
|
||||||
|
- Create region: with the Pointer tool, double‑click; or hold Ctrl/Cmd and click. With the Pencil tool, single‑click.
|
||||||
|
- Move/resize: drag the body to move; drag edges to resize.
|
||||||
|
- Open Piano Roll via the small pencil at a region’s top‑left.
|
||||||
|
|
||||||
|
- Piano Roll (MIDI notes)
|
||||||
|
- Tools: Select vs Pencil.
|
||||||
|
- Create notes: double‑click or Ctrl/Cmd+click (Select); single‑click (Pencil).
|
||||||
|
- Select notes: click; Shift+click for multi‑select; drag to box‑select.
|
||||||
|
- Move/resize: drag note body to move selected notes; drag edges to resize.
|
||||||
|
- Close the piano roll with X or ESC.
|
||||||
|
|
||||||
|
- Snapping and Quantize
|
||||||
|
- Set snapping from the NO SNAP menu (top‑right).
|
||||||
|
- Quantize timing with Qua. Pos. (start) and Qua. Len. (length).
|
||||||
|
|
||||||
|
- Playback & Playhead
|
||||||
|
- Back to beginning; Play/Pause from the toolbar.
|
||||||
|
- Set the playhead by clicking bar numbers in the main grid; in Piano Roll, click the header timeline (respects snapping).
|
||||||
|
- Change BPM, time signature, and key signature via the toolbar readouts.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Note
|
||||||
|
|
||||||
|
Due to security reasons, if you are using K.G.Studio from a non-local hosted environment, we won't persist your API Key in your browser's IndexedDB in order to prevent potential XSS attack, you will need to input your API Key every time you start K.G.Studio.
|
||||||
|
|
||||||
|
### Disclaimer
|
||||||
|
|
||||||
|
K.G.Studio does not provide or host any of the models listed above, nor is it affiliated with any model provider. All data is stored locally on your device; K.G.Studio does not collect or transmit your data. You are solely responsible for any data you provide to third-party model providers.
|
||||||
|
|
||||||
|
We hope you enjoy using K.G.Studio Musician Assistant!
|
||||||
@@ -6,5 +6,6 @@ Tips:
|
|||||||
- Use `/clear` anytime to reset the chat.
|
- Use `/clear` anytime to reset the chat.
|
||||||
- Ask me to create tracks, regions, or MIDI notes, and I'll help orchestrate tool actions.
|
- Ask me to create tracks, regions, or MIDI notes, and I'll help orchestrate tool actions.
|
||||||
- Type `/welcome` to view this message again.
|
- Type `/welcome` to view this message again.
|
||||||
|
- Type `/help` to view the help message.
|
||||||
|
|
||||||
Have fun composing with K.G.Studio Musician Assistant.
|
Have fun composing with K.G.Studio Musician Assistant.
|
||||||
+18
-46
@@ -1,52 +1,24 @@
|
|||||||
## Welcome
|
## Welcome
|
||||||
|
Welcome to **K.G.Studio Musician Assistant** — your in‑browser AI partner for composing and arranging.
|
||||||
|
|
||||||
Welcome to **K.G.Studio Musician Assistant**—your AI-powered agent for music composition and arrangement. Harnessing advanced language models, I am here to help you create, arrange, and refine your musical ideas with ease and intelligence.
|
### Quick Start
|
||||||
|
**Set up K.G.Studio Musician Assistant**
|
||||||
|
- [Click here to get a free OpenRouter API Key](https://openrouter.ai/keys) (you may need an OpenRouter account).
|
||||||
|
- In **Settings ⚙️ → General → LLM Provider**, select **OpenAI Compatible**.
|
||||||
|
- In **OpenAI Compatible Server → Key**, paste your key. Note: on non‑localhost, your key isn’t persisted; you’ll re‑enter it after refresh. Keep it safe.
|
||||||
|
- In **OpenAI Compatible Server → Model**, enter `qwen/qwen3-30b-a3b:free`. Note: non‑free models may require billing.
|
||||||
|
- In **OpenAI Compatible Server → Base URL**, enter `https://openrouter.ai/api/v1/chat/completions`.
|
||||||
|
|
||||||
To get started, you will need to configure an LLM (Large Language Model) Provider in the application settings. Please follow the instructions below to ensure proper setup:
|
*(Alternatively, you can use the official OpenAI API, or any OpenAI‑compatible service)*
|
||||||
|
|
||||||
---
|
**Basic DAW operations**
|
||||||
|
- Double‑click (or hold Ctrl/Cmd and click) on a track to create a region.
|
||||||
### Configuring Your LLM Provider
|
- Drag region edges to resize; drag the body to move.
|
||||||
|
- Click the small pencil at a region’s top‑left to open the piano roll.
|
||||||
Navigate to **Settings ⚙️ → General → LLM Provider**. Depending on your chosen provider, you will need to supply the appropriate API Key and, if applicable, a custom base URL (for non-official OpenAI-compatible services such as Ollama, OpenRouter, etc.).
|
- In the piano roll, double‑click (or Ctrl/Cmd+click) to create a note.
|
||||||
|
- Click to select; Shift+click for multi‑select; drag to box‑select.
|
||||||
---
|
- Drag note edges to resize; drag the note body to move selected notes.
|
||||||
|
- Use Snapping in the piano roll toolbar (top‑right) to quantize to grid.
|
||||||
### Using the OpenAI GPT Model Series
|
|
||||||
|
|
||||||
1. Obtain an OpenAI API Key from [**OpenAI**](https://platform.openai.com/account/api-keys). You may need to create an account and add a payment method to generate an API Key.
|
|
||||||
2. In **Settings ⚙️ → General → LLM Provider**, select **OpenAI** as your provider.
|
|
||||||
3. Enter your API Key in **OpenAI → Key**.
|
|
||||||
4. Select your preferred model from the **OpenAI → Model** dropdown. For optimal performance, we recommend `gpt-5` or `gpt-4o`.
|
|
||||||
5. Optionally, choose whether to enable Flex Mode in **OpenAI → Flex Mode**. Flex Mode offers discounted pricing, but may result in slower response times.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Using OpenRouter
|
|
||||||
|
|
||||||
OpenRouter is a platform that provides unified access to a wide range of language models—including free options—from various providers. This makes it easy to experiment and find the model that best suits your needs.
|
|
||||||
|
|
||||||
1. Obtain an API Key from [**OpenRouter**](https://openrouter.ai/keys). Registration is required; for paid models, a payment method may also be necessary.
|
|
||||||
2. In **Settings ⚙️ → General → LLM Provider**, select **OpenAI Compatible** as your provider.
|
|
||||||
3. Enter your API Key in **OpenAI Compatible Server → Key**.
|
|
||||||
4. Browse available models on the [**OpenRouter Models Page**](https://openrouter.ai/models). Use the "Prompt Pricing" filter to identify free models.
|
|
||||||
**Note:** Each model provider may have different data retention and privacy policies. Please review these policies before use.
|
|
||||||
5. Enter your chosen model name in **OpenAI Compatible Server → Model**. Recommended model series include:
|
|
||||||
- `Google: gemini-2.5-pro`
|
|
||||||
- `Anthropic: claude-4-sonnet`
|
|
||||||
- `DeepSeek: deepseek-r1` (free: `deepseek/deepseek-r1-0528:free`)
|
|
||||||
- `DeepSeek: deepseek-v3` (free: `deepseek/deepseek-chat-v3-0324:free`)
|
|
||||||
- `Qwen: qwen3-235b-a22b` (free: `qwen/qwen3-235b-a22b:free`)
|
|
||||||
6. Input the base URL `https://openrouter.ai/api/v1/chat/completions` of the OpenAI Compatible Server in **OpenAI Compatible Server → Base URL**.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Note
|
|
||||||
|
|
||||||
Due to security reasons, if you are using K.G.Studio from a non-local hosted environment, we won't persist your API Key in your browser's IndexedDB in order to prevent potential XSS attack, you will need to input your API Key every time you start K.G.Studio.
|
|
||||||
|
|
||||||
### Disclaimer
|
### Disclaimer
|
||||||
|
K.G.Studio is not affiliated with any LLM provider. All data is stored locally on your device; you are responsible for any data you send to third‑party services.
|
||||||
K.G.Studio does not provide or host any of the models listed above, nor is it affiliated with any model provider. All data is stored locally on your device; K.G.Studio does not collect or transmit your data. You are solely responsible for any data you provide to third-party model providers.
|
|
||||||
|
|
||||||
We hope you enjoy using K.G.Studio Musician Assistant!
|
|
||||||
@@ -54,6 +54,9 @@
|
|||||||
"chatbox": {
|
"chatbox": {
|
||||||
"default_open": true
|
"default_open": true
|
||||||
},
|
},
|
||||||
|
"audio": {
|
||||||
|
"enable_audio_capture_for_screen_sharing": false
|
||||||
|
},
|
||||||
"templates": {
|
"templates": {
|
||||||
"custom_instructions": ""
|
"custom_instructions": ""
|
||||||
}
|
}
|
||||||
|
|||||||
+26
-3
@@ -669,9 +669,9 @@ body {
|
|||||||
color: #e0e0e0;
|
color: #e0e0e0;
|
||||||
}
|
}
|
||||||
|
|
||||||
.export-dropdown .quant-dropdown {
|
.chatbox-export-dropdown .quant-dropdown {
|
||||||
width: 200px;
|
width: 250px;
|
||||||
left: 0;
|
left: -200px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.key-signature-dropdown .quant-dropdown {
|
.key-signature-dropdown .quant-dropdown {
|
||||||
@@ -1039,6 +1039,10 @@ body {
|
|||||||
flex-shrink: 0;
|
flex-shrink: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.chatbox.is-hidden {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
|
||||||
/* Instrument Selection Panel */
|
/* Instrument Selection Panel */
|
||||||
.instrument-selection {
|
.instrument-selection {
|
||||||
display: flex;
|
display: flex;
|
||||||
@@ -1197,6 +1201,25 @@ body {
|
|||||||
border-radius: 3px;
|
border-radius: 3px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ChatBox export button wrapper and dropdown positioning */
|
||||||
|
.chatbox-export-wrapper {
|
||||||
|
position: relative;
|
||||||
|
display: inline-block;
|
||||||
|
}
|
||||||
|
|
||||||
|
.chatbox-export-btn {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 4px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.chatbox-export-dropdown-anchor {
|
||||||
|
position: absolute;
|
||||||
|
top: 100%;
|
||||||
|
left: 0;
|
||||||
|
z-index: 10000;
|
||||||
|
}
|
||||||
|
|
||||||
.chatbox-header h3 {
|
.chatbox-header h3 {
|
||||||
color: #e0e0e0;
|
color: #e0e0e0;
|
||||||
font-size: 12px;
|
font-size: 12px;
|
||||||
|
|||||||
@@ -93,31 +93,6 @@ export class AgentCore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Process user input and get complete response (non-streaming)
|
|
||||||
*/
|
|
||||||
async processUserInputComplete(userInput: string): Promise<string> {
|
|
||||||
if (!this.llmProvider) {
|
|
||||||
throw new Error('No LLM provider configured');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add user message to state
|
|
||||||
this.agentState.addMessage('user', userInput);
|
|
||||||
|
|
||||||
// Get system prompt with current context
|
|
||||||
const systemPrompt = await SystemPrompts.getSystemPromptWithContext();
|
|
||||||
|
|
||||||
// Get full conversation history with preserved roles
|
|
||||||
const conversationHistory = this.agentState.getMessages();
|
|
||||||
|
|
||||||
// Generate complete response with full conversation context
|
|
||||||
const response = await this.llmProvider.generateCompletion(conversationHistory, systemPrompt);
|
|
||||||
|
|
||||||
// Add assistant response to state
|
|
||||||
this.agentState.addMessage('assistant', response.content);
|
|
||||||
|
|
||||||
return response.content;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Abort the current streaming request and clean up messages
|
* Abort the current streaming request and clean up messages
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { LLMProvider } from './LLMProvider';
|
import { LLMProvider } from './LLMProvider';
|
||||||
import type { StreamChunk, LLMResponse } from './StreamingTypes';
|
import type { StreamChunk } from './StreamingTypes';
|
||||||
import type { Message } from '../core/AgentState';
|
import type { Message } from '../core/AgentState';
|
||||||
import { ConfigManager } from '../../core/config/ConfigManager';
|
import { ConfigManager } from '../../core/config/ConfigManager';
|
||||||
|
|
||||||
@@ -159,71 +159,4 @@ export class ClaudeProvider extends LLMProvider {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async generateCompletion(
|
|
||||||
messages: Message[],
|
|
||||||
systemPrompt?: string,
|
|
||||||
tools?: Record<string, unknown>[]
|
|
||||||
): Promise<LLMResponse> {
|
|
||||||
const { system, messages: claudeMessages } = this.convertMessages(messages, systemPrompt);
|
|
||||||
|
|
||||||
const requestBody: {
|
|
||||||
model: string;
|
|
||||||
max_tokens: number;
|
|
||||||
messages: Array<{ role: 'user' | 'assistant'; content: string }>;
|
|
||||||
stream: boolean;
|
|
||||||
system?: string;
|
|
||||||
tools?: Record<string, unknown>[];
|
|
||||||
} = {
|
|
||||||
model: this.model,
|
|
||||||
max_tokens: 8192,
|
|
||||||
messages: claudeMessages,
|
|
||||||
stream: false
|
|
||||||
};
|
|
||||||
|
|
||||||
if (system) {
|
|
||||||
requestBody.system = system;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (tools && tools.length > 0) {
|
|
||||||
requestBody.tools = tools;
|
|
||||||
}
|
|
||||||
|
|
||||||
const response = await fetch(this.apiEndpoint, {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
'x-api-key': this.apiKey,
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
'anthropic-version': '2023-06-01'
|
|
||||||
},
|
|
||||||
body: JSON.stringify(requestBody),
|
|
||||||
mode: 'cors'
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
throw new Error(`Claude API error: ${response.status} ${response.statusText}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const data = await response.json();
|
|
||||||
|
|
||||||
// Extract content from Claude's response format
|
|
||||||
const content = data.content
|
|
||||||
?.filter((block: { type: string }) => block.type === 'text')
|
|
||||||
?.map((block: { text: string }) => block.text)
|
|
||||||
?.join('') || '';
|
|
||||||
|
|
||||||
// Extract tool calls if present
|
|
||||||
const toolCalls = data.content
|
|
||||||
?.filter((block: { type: string }) => block.type === 'tool_use')
|
|
||||||
?.map((block: { id: string; name: string; input: Record<string, unknown> }) => ({
|
|
||||||
id: block.id,
|
|
||||||
name: block.name,
|
|
||||||
parameters: block.input
|
|
||||||
})) || [];
|
|
||||||
|
|
||||||
return {
|
|
||||||
content,
|
|
||||||
toolCalls: toolCalls.length > 0 ? toolCalls : undefined,
|
|
||||||
finished: data.stop_reason === 'end_turn' || data.stop_reason === 'tool_use'
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
import { LLMProvider } from './LLMProvider';
|
import { LLMProvider } from './LLMProvider';
|
||||||
import type { StreamChunk, LLMResponse } from './StreamingTypes';
|
import type { StreamChunk } from './StreamingTypes';
|
||||||
import type { Message } from '../core/AgentState';
|
import type { Message } from '../core/AgentState';
|
||||||
import { ConfigManager } from '../../core/config/ConfigManager';
|
import { ConfigManager } from '../../core/config/ConfigManager';
|
||||||
|
|
||||||
@@ -180,66 +180,4 @@ export class GeminiProvider extends LLMProvider {
|
|||||||
reader.releaseLock();
|
reader.releaseLock();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async generateCompletion(
|
|
||||||
messages: Message[],
|
|
||||||
systemPrompt?: string,
|
|
||||||
tools?: Record<string, unknown>[]
|
|
||||||
): Promise<LLMResponse> {
|
|
||||||
const { systemInstruction, contents } = this.convertMessages(messages, systemPrompt);
|
|
||||||
|
|
||||||
const requestBody: {
|
|
||||||
contents: Array<{ role: 'user' | 'model'; parts: Array<{ text: string }> }>;
|
|
||||||
generationConfig: { temperature: number; maxOutputTokens: number };
|
|
||||||
systemInstruction?: { parts: Array<{ text: string }> };
|
|
||||||
tools?: Record<string, unknown>[];
|
|
||||||
} = {
|
|
||||||
contents,
|
|
||||||
generationConfig: {
|
|
||||||
temperature: 0.7,
|
|
||||||
maxOutputTokens: 8192,
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
if (systemInstruction) {
|
|
||||||
requestBody.systemInstruction = systemInstruction;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (tools && tools.length > 0) {
|
|
||||||
requestBody.tools = tools;
|
|
||||||
}
|
|
||||||
|
|
||||||
const response = await fetch(`${this.apiEndpoint}:generateContent?key=${this.apiKey}`, {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
},
|
|
||||||
body: JSON.stringify(requestBody),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
throw new Error(`Gemini API error: ${response.status} ${response.statusText}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const data = await response.json();
|
|
||||||
|
|
||||||
// Extract content from Gemini's response format
|
|
||||||
const candidate = data.candidates?.[0];
|
|
||||||
const content = candidate?.content?.parts?.[0]?.text || '';
|
|
||||||
|
|
||||||
// Extract tool calls if present (Gemini format)
|
|
||||||
const toolCalls = candidate?.content?.parts
|
|
||||||
?.filter((part: { functionCall?: unknown }) => part.functionCall)
|
|
||||||
?.map((part: { functionCall: { name: string; args: Record<string, unknown> } }) => ({
|
|
||||||
id: `tool_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`, // Generate ID
|
|
||||||
name: part.functionCall.name,
|
|
||||||
parameters: part.functionCall.args
|
|
||||||
})) || [];
|
|
||||||
|
|
||||||
return {
|
|
||||||
content,
|
|
||||||
toolCalls: toolCalls.length > 0 ? toolCalls : undefined,
|
|
||||||
finished: candidate?.finishReason === 'STOP' || candidate?.finishReason === 'MAX_TOKENS'
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
import type { StreamChunk, LLMResponse } from './StreamingTypes';
|
import type { StreamChunk } from './StreamingTypes';
|
||||||
import type { Message } from '../core/AgentState';
|
import type { Message } from '../core/AgentState';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -18,16 +18,4 @@ export abstract class LLMProvider {
|
|||||||
systemPrompt?: string,
|
systemPrompt?: string,
|
||||||
tools?: Record<string, unknown>[]
|
tools?: Record<string, unknown>[]
|
||||||
): AsyncIterableIterator<StreamChunk>;
|
): AsyncIterableIterator<StreamChunk>;
|
||||||
|
|
||||||
/**
|
|
||||||
* Generate a complete response from the LLM (non-streaming)
|
|
||||||
* @param messages The full conversation history with preserved roles
|
|
||||||
* @param systemPrompt The system prompt (optional, can be included in messages)
|
|
||||||
* @param tools Available tools (optional for now)
|
|
||||||
*/
|
|
||||||
abstract generateCompletion(
|
|
||||||
messages: Message[],
|
|
||||||
systemPrompt?: string,
|
|
||||||
tools?: Record<string, unknown>[]
|
|
||||||
): Promise<LLMResponse>;
|
|
||||||
}
|
}
|
||||||
+92
-317
@@ -1,8 +1,9 @@
|
|||||||
import { LLMProvider } from './LLMProvider';
|
import { LLMProvider } from './LLMProvider';
|
||||||
import type { StreamChunk, LLMResponse } from './StreamingTypes';
|
import type { StreamChunk } from './StreamingTypes';
|
||||||
import type { Message } from '../core/AgentState';
|
import type { Message } from '../core/AgentState';
|
||||||
import { ConfigManager } from '../../core/config/ConfigManager';
|
import { ConfigManager } from '../../core/config/ConfigManager';
|
||||||
import { URL_CONSTANTS } from '../../constants/coreConstants';
|
import { URL_CONSTANTS } from '../../constants/coreConstants';
|
||||||
|
import { LLM_PROTOCOL } from '../../constants/llmConstants';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* OpenAI API provider implementation
|
* OpenAI API provider implementation
|
||||||
@@ -16,6 +17,78 @@ export class OpenAIProvider extends LLMProvider {
|
|||||||
super();
|
super();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build OpenAI-compatible messages array from input messages and system prompt
|
||||||
|
*/
|
||||||
|
private buildRequestMessages(messages: Message[], systemPrompt?: string): Array<{ role: string; content: string }> {
|
||||||
|
const openAIMessages: Array<{ role: string; content: string }> = [];
|
||||||
|
|
||||||
|
// Add system prompt if provided
|
||||||
|
if (systemPrompt) {
|
||||||
|
openAIMessages.push({ role: 'system', content: systemPrompt });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add conversation history with preserved roles
|
||||||
|
openAIMessages.push(...messages.map(msg => ({
|
||||||
|
role: msg.role,
|
||||||
|
content: msg.content
|
||||||
|
})));
|
||||||
|
|
||||||
|
return openAIMessages;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create API request with proper headers and body
|
||||||
|
*/
|
||||||
|
private async createApiRequest(messages: Array<{ role: string; content: string }>, config: ReturnType<typeof this.getCurrentConfig>, streaming: boolean): Promise<Response> {
|
||||||
|
const response = await fetch(config.apiEndpoint, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Authorization': `Bearer ${config.apiKey}`,
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
model: config.model,
|
||||||
|
...(config.flexMode && !config.isCompatibleProvider ? { service_tier: 'flex' } : {}),
|
||||||
|
messages,
|
||||||
|
stream: streaming,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const errorText = await response.text().catch(() => 'Unknown error');
|
||||||
|
throw new Error(`${this.name} API request failed (${response.status}): ${response.statusText}. ${errorText}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Process thinking and content chunks, yielding appropriate StreamChunks
|
||||||
|
*/
|
||||||
|
private async *processContentChunk(
|
||||||
|
thinking: string | undefined,
|
||||||
|
content: string | undefined,
|
||||||
|
lastSegmentType: { current: 'thinking' | 'content' | null }
|
||||||
|
): AsyncIterableIterator<StreamChunk> {
|
||||||
|
if (typeof thinking === 'string' && thinking.length > 0) {
|
||||||
|
if (lastSegmentType.current && lastSegmentType.current !== 'thinking') {
|
||||||
|
yield { type: 'text', content: LLM_PROTOCOL.SEGMENT_SEPARATOR };
|
||||||
|
}
|
||||||
|
yield { type: 'text', content: thinking };
|
||||||
|
lastSegmentType.current = 'thinking';
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof content === 'string' && content.length > 0) {
|
||||||
|
if (lastSegmentType.current && lastSegmentType.current !== 'content') {
|
||||||
|
yield { type: 'text', content: LLM_PROTOCOL.SEGMENT_SEPARATOR };
|
||||||
|
}
|
||||||
|
yield { type: 'text', content: content };
|
||||||
|
lastSegmentType.current = 'content';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get current configuration values from ConfigManager
|
* Get current configuration values from ConfigManager
|
||||||
*/
|
*/
|
||||||
@@ -50,7 +123,7 @@ export class OpenAIProvider extends LLMProvider {
|
|||||||
*/
|
*/
|
||||||
private detectStreamFormat(firstChunk: string): boolean {
|
private detectStreamFormat(firstChunk: string): boolean {
|
||||||
// If it starts with "data: ", it's OpenAI SSE format
|
// If it starts with "data: ", it's OpenAI SSE format
|
||||||
if (firstChunk.trim().startsWith('data: ')) {
|
if (firstChunk.trim().startsWith(LLM_PROTOCOL.SSE_DATA_PREFIX)) {
|
||||||
return false; // Not Ollama format
|
return false; // Not Ollama format
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -66,30 +139,15 @@ export class OpenAIProvider extends LLMProvider {
|
|||||||
/**
|
/**
|
||||||
* Parse Ollama's raw JSON chunk format
|
* Parse Ollama's raw JSON chunk format
|
||||||
*/
|
*/
|
||||||
private parseOllamaChunk(chunk: string): { thinking?: string; content?: string; isDone?: boolean; toolCalls?: Array<{ name: string; arguments: Record<string, unknown> }> } {
|
private parseOllamaChunk(chunk: string): { thinking?: string; content?: string; isDone?: boolean } {
|
||||||
try {
|
try {
|
||||||
const json = JSON.parse(chunk.trim());
|
const json = JSON.parse(chunk.trim());
|
||||||
const thinking: string | undefined = json.message?.thinking;
|
const thinking: string | undefined = json.message?.thinking;
|
||||||
const content: string | undefined = json.message?.content || json.response; // Handle both chat and completion formats
|
const content: string | undefined = json.message?.content || json.response; // Handle both chat and completion formats
|
||||||
type WireToolCall = { function?: { name?: unknown; arguments?: unknown } };
|
|
||||||
const toolCallsRaw: unknown[] | undefined = json.message?.tool_calls as unknown[] | undefined;
|
|
||||||
const toolCalls = Array.isArray(toolCallsRaw)
|
|
||||||
? (toolCallsRaw
|
|
||||||
.map((tc: unknown) => {
|
|
||||||
const wire = tc as WireToolCall;
|
|
||||||
const fn = wire?.function;
|
|
||||||
if (!fn || typeof fn.name !== 'string') return null;
|
|
||||||
const args = fn.arguments;
|
|
||||||
if (args === null || typeof args !== 'object' || Array.isArray(args)) return null;
|
|
||||||
return { name: fn.name, arguments: args as Record<string, unknown> };
|
|
||||||
})
|
|
||||||
.filter((v): v is { name: string; arguments: Record<string, unknown> } => v !== null))
|
|
||||||
: undefined;
|
|
||||||
return {
|
return {
|
||||||
thinking,
|
thinking,
|
||||||
content,
|
content,
|
||||||
isDone: json.done === true,
|
isDone: json.done === true
|
||||||
toolCalls
|
|
||||||
};
|
};
|
||||||
} catch {
|
} catch {
|
||||||
return {}; // Invalid JSON, return empty object
|
return {}; // Invalid JSON, return empty object
|
||||||
@@ -99,13 +157,13 @@ export class OpenAIProvider extends LLMProvider {
|
|||||||
/**
|
/**
|
||||||
* Parse OpenAI's SSE format chunk
|
* Parse OpenAI's SSE format chunk
|
||||||
*/
|
*/
|
||||||
private parseOpenAIChunk(line: string): { thinking?: string; content?: string; isDone?: boolean; toolCallDelta?: Array<{ index?: number; function?: { name?: string; arguments?: string } }> } {
|
private parseOpenAIChunk(line: string): { thinking?: string; content?: string; isDone?: boolean } {
|
||||||
if (!line.startsWith('data: ')) {
|
if (!line.startsWith(LLM_PROTOCOL.SSE_DATA_PREFIX)) {
|
||||||
return {};
|
return {};
|
||||||
}
|
}
|
||||||
|
|
||||||
const data = line.slice(6);
|
const data = line.slice(LLM_PROTOCOL.SSE_DATA_PREFIX.length);
|
||||||
if (data === '[DONE]') {
|
if (data === LLM_PROTOCOL.SSE_DONE_MARKER) {
|
||||||
return { isDone: true };
|
return { isDone: true };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -114,11 +172,7 @@ export class OpenAIProvider extends LLMProvider {
|
|||||||
const delta = json.choices?.[0]?.delta;
|
const delta = json.choices?.[0]?.delta;
|
||||||
const thinking: string | undefined = delta?.thinking; // Some providers may stream "thinking"
|
const thinking: string | undefined = delta?.thinking; // Some providers may stream "thinking"
|
||||||
const content: string | undefined = delta?.content;
|
const content: string | undefined = delta?.content;
|
||||||
const tcd = delta?.tool_calls;
|
return { thinking, content, isDone: false };
|
||||||
const toolCallDelta: Array<{ index?: number; function?: { name?: string; arguments?: string } }> | undefined = Array.isArray(tcd)
|
|
||||||
? (tcd as Array<{ index?: number; function?: { name?: string; arguments?: string } }>)
|
|
||||||
: undefined;
|
|
||||||
return { thinking, content, isDone: false, toolCallDelta };
|
|
||||||
} catch {
|
} catch {
|
||||||
return {}; // Skip invalid JSON lines
|
return {}; // Skip invalid JSON lines
|
||||||
}
|
}
|
||||||
@@ -126,77 +180,22 @@ export class OpenAIProvider extends LLMProvider {
|
|||||||
|
|
||||||
async *generateStream(
|
async *generateStream(
|
||||||
messages: Message[],
|
messages: Message[],
|
||||||
systemPrompt?: string,
|
systemPrompt?: string
|
||||||
tools?: Record<string, unknown>[]
|
|
||||||
): AsyncIterableIterator<StreamChunk> {
|
): AsyncIterableIterator<StreamChunk> {
|
||||||
// Get fresh config values
|
|
||||||
const config = this.getCurrentConfig();
|
const config = this.getCurrentConfig();
|
||||||
|
const requestMessages = this.buildRequestMessages(messages, systemPrompt);
|
||||||
// Build OpenAI messages array with role preservation
|
const response = await this.createApiRequest(requestMessages, config, true);
|
||||||
const openAIMessages: Array<{ role: string; content: string }> = [];
|
|
||||||
|
|
||||||
// Add system prompt if provided
|
|
||||||
if (systemPrompt) {
|
|
||||||
openAIMessages.push({ role: 'system', content: systemPrompt });
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add conversation history with preserved roles
|
|
||||||
openAIMessages.push(...messages.map(msg => ({
|
|
||||||
role: msg.role,
|
|
||||||
content: msg.content
|
|
||||||
})));
|
|
||||||
|
|
||||||
const response = await fetch(config.apiEndpoint, {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
'Authorization': `Bearer ${config.apiKey}`,
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
},
|
|
||||||
body: JSON.stringify({
|
|
||||||
model: config.model,
|
|
||||||
...(config.flexMode && !config.isCompatibleProvider ? { service_tier: 'flex' } : {}),
|
|
||||||
messages: openAIMessages,
|
|
||||||
stream: true,
|
|
||||||
tools: tools || undefined
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
throw new Error(`OpenAI API error: ${response.status} ${response.statusText}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const reader = response.body?.getReader();
|
const reader = response.body?.getReader();
|
||||||
if (!reader) {
|
if (!reader) {
|
||||||
throw new Error('Failed to get response reader');
|
throw new Error(`${this.name} streaming: Failed to get response reader from API response`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const decoder = new TextDecoder();
|
const decoder = new TextDecoder();
|
||||||
let buffer = '';
|
let buffer = '';
|
||||||
let firstChunkProcessed = false;
|
let firstChunkProcessed = false;
|
||||||
let lastSegmentType: 'thinking' | 'content' | null = null;
|
const lastSegmentType = { current: null as 'thinking' | 'content' | null };
|
||||||
const pendingFunctionCalls: Array<{ name: string; arguments: Record<string, unknown> }> = [];
|
|
||||||
const openAIToolCallBuilders: Record<number, { name?: string; argumentsText: string }> = {};
|
|
||||||
|
|
||||||
const escapeXml = (text: string): string =>
|
|
||||||
String(text)
|
|
||||||
.replace(/&/g, '&')
|
|
||||||
.replace(/</g, '<')
|
|
||||||
.replace(/>/g, '>')
|
|
||||||
.replace(/"/g, '"')
|
|
||||||
.replace(/'/g, ''');
|
|
||||||
|
|
||||||
const functionCallToXml = (name: string, args: Record<string, unknown>): string => {
|
|
||||||
const keys = Object.keys(args);
|
|
||||||
const inner = keys
|
|
||||||
.map((k) => {
|
|
||||||
const value = (args as Record<string, unknown>)[k];
|
|
||||||
const text = typeof value === 'string' ? value : JSON.stringify(value);
|
|
||||||
return `<${k}>${escapeXml(text)}</${k}>`;
|
|
||||||
})
|
|
||||||
.join('\n');
|
|
||||||
return `<${name}>\n${inner}\n</${name}>`;
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
while (true) {
|
while (true) {
|
||||||
const { done, value } = await reader.read();
|
const { done, value } = await reader.read();
|
||||||
@@ -219,244 +218,20 @@ export class OpenAIProvider extends LLMProvider {
|
|||||||
firstChunkProcessed = true;
|
firstChunkProcessed = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.isOllamaFormat) {
|
const { thinking, content, isDone } = this.isOllamaFormat
|
||||||
const { thinking, content, isDone, toolCalls } = this.parseOllamaChunk(trimmedLine);
|
? this.parseOllamaChunk(trimmedLine)
|
||||||
if (isDone) {
|
: this.parseOpenAIChunk(trimmedLine);
|
||||||
// Append any pending or current tool calls as XML before finishing
|
|
||||||
const allToolCalls = [
|
|
||||||
...pendingFunctionCalls,
|
|
||||||
...(toolCalls || [])
|
|
||||||
];
|
|
||||||
if (allToolCalls.length > 0) {
|
|
||||||
const xmlBlocks = allToolCalls
|
|
||||||
.map((tc) => `\n${functionCallToXml(tc.name, tc.arguments)}\n`)
|
|
||||||
.join('');
|
|
||||||
yield { type: 'text', content: xmlBlocks };
|
|
||||||
}
|
|
||||||
yield { type: 'done', content: '' };
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (typeof thinking === 'string' && thinking.length > 0) {
|
|
||||||
if (lastSegmentType && lastSegmentType !== 'thinking') {
|
|
||||||
yield { type: 'text', content: '\n\n' };
|
|
||||||
}
|
|
||||||
yield { type: 'text', content: thinking };
|
|
||||||
lastSegmentType = 'thinking';
|
|
||||||
}
|
|
||||||
|
|
||||||
if (typeof content === 'string' && content.length > 0) {
|
|
||||||
if (lastSegmentType && lastSegmentType !== 'content') {
|
|
||||||
yield { type: 'text', content: '\n\n' };
|
|
||||||
}
|
|
||||||
yield { type: 'text', content: content };
|
|
||||||
lastSegmentType = 'content';
|
|
||||||
}
|
|
||||||
|
|
||||||
if (Array.isArray(toolCalls) && toolCalls.length > 0) {
|
|
||||||
// Accumulate and append at the end of stream
|
|
||||||
pendingFunctionCalls.push(...toolCalls);
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const { thinking, content, isDone, toolCallDelta } = this.parseOpenAIChunk(trimmedLine);
|
|
||||||
if (isDone) {
|
if (isDone) {
|
||||||
// Finalize any accumulated OpenAI tool calls and emit XML
|
|
||||||
const finalizedToolCalls: Array<{ name: string; arguments: Record<string, unknown> }> = [];
|
|
||||||
for (const indexStr of Object.keys(openAIToolCallBuilders)) {
|
|
||||||
const idx = Number(indexStr);
|
|
||||||
const builder = openAIToolCallBuilders[idx];
|
|
||||||
if (!builder || !builder.name) continue;
|
|
||||||
let argsObj: Record<string, unknown> | null = null;
|
|
||||||
if (builder.argumentsText) {
|
|
||||||
try {
|
|
||||||
argsObj = JSON.parse(builder.argumentsText);
|
|
||||||
} catch {
|
|
||||||
argsObj = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (argsObj) {
|
|
||||||
finalizedToolCalls.push({ name: builder.name, arguments: argsObj });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const allToolCalls = [...pendingFunctionCalls, ...finalizedToolCalls];
|
|
||||||
if (allToolCalls.length > 0) {
|
|
||||||
const xmlBlocks = allToolCalls
|
|
||||||
.map((tc) => `\n${functionCallToXml(tc.name, tc.arguments)}\n`)
|
|
||||||
.join('');
|
|
||||||
yield { type: 'text', content: xmlBlocks };
|
|
||||||
}
|
|
||||||
yield { type: 'done', content: '' };
|
yield { type: 'done', content: '' };
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (typeof thinking === 'string' && thinking.length > 0) {
|
|
||||||
if (lastSegmentType && lastSegmentType !== 'thinking') {
|
|
||||||
yield { type: 'text', content: '\n\n' };
|
|
||||||
}
|
|
||||||
yield { type: 'text', content: thinking };
|
|
||||||
lastSegmentType = 'thinking';
|
|
||||||
}
|
|
||||||
|
|
||||||
if (typeof content === 'string' && content.length > 0) {
|
yield* this.processContentChunk(thinking, content, lastSegmentType);
|
||||||
if (lastSegmentType && lastSegmentType !== 'content') {
|
|
||||||
yield { type: 'text', content: '\n\n' };
|
|
||||||
}
|
|
||||||
yield { type: 'text', content };
|
|
||||||
lastSegmentType = 'content';
|
|
||||||
}
|
|
||||||
|
|
||||||
if (Array.isArray(toolCallDelta) && toolCallDelta.length > 0) {
|
|
||||||
for (const tc of toolCallDelta) {
|
|
||||||
const index: number = typeof tc.index === 'number' ? tc.index : 0;
|
|
||||||
if (!openAIToolCallBuilders[index]) {
|
|
||||||
openAIToolCallBuilders[index] = { argumentsText: '' };
|
|
||||||
}
|
|
||||||
const fn = tc.function;
|
|
||||||
if (fn) {
|
|
||||||
if (typeof fn.name === 'string') {
|
|
||||||
openAIToolCallBuilders[index].name = fn.name;
|
|
||||||
}
|
|
||||||
if (typeof fn.arguments === 'string') {
|
|
||||||
openAIToolCallBuilders[index].argumentsText += fn.arguments;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
reader.releaseLock();
|
reader.releaseLock();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async generateCompletion(
|
|
||||||
messages: Message[],
|
|
||||||
systemPrompt?: string,
|
|
||||||
tools?: Record<string, unknown>[]
|
|
||||||
): Promise<LLMResponse> {
|
|
||||||
// Get fresh config values
|
|
||||||
const config = this.getCurrentConfig();
|
|
||||||
|
|
||||||
// Build OpenAI messages array with role preservation
|
|
||||||
const openAIMessages: Array<{ role: string; content: string }> = [];
|
|
||||||
|
|
||||||
// Add system prompt if provided
|
|
||||||
if (systemPrompt) {
|
|
||||||
openAIMessages.push({ role: 'system', content: systemPrompt });
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add conversation history with preserved roles
|
|
||||||
openAIMessages.push(...messages.map(msg => ({
|
|
||||||
role: msg.role,
|
|
||||||
content: msg.content
|
|
||||||
})));
|
|
||||||
|
|
||||||
const response = await fetch(config.apiEndpoint, {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
'Authorization': `Bearer ${config.apiKey}`,
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
},
|
|
||||||
body: JSON.stringify({
|
|
||||||
model: config.model,
|
|
||||||
...(config.flexMode && !config.isCompatibleProvider ? { service_tier: 'flex' } : {}),
|
|
||||||
messages: openAIMessages,
|
|
||||||
stream: false,
|
|
||||||
tools: tools || undefined
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
throw new Error(`OpenAI API error: ${response.status} ${response.statusText}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const data = await response.json();
|
|
||||||
|
|
||||||
// Handle different response formats
|
|
||||||
const escapeXml = (text: string): string =>
|
|
||||||
String(text)
|
|
||||||
.replace(/&/g, '&')
|
|
||||||
.replace(/</g, '<')
|
|
||||||
.replace(/>/g, '>')
|
|
||||||
.replace(/"/g, '"')
|
|
||||||
.replace(/'/g, ''');
|
|
||||||
|
|
||||||
const functionCallToXml = (name: string, args: Record<string, unknown>): string => {
|
|
||||||
const keys = Object.keys(args);
|
|
||||||
const inner = keys
|
|
||||||
.map((k) => {
|
|
||||||
const value = (args as Record<string, unknown>)[k];
|
|
||||||
const text = typeof value === 'string' ? value : JSON.stringify(value);
|
|
||||||
return `<${k}>${escapeXml(text)}</${k}>`;
|
|
||||||
})
|
|
||||||
.join('\n');
|
|
||||||
return `<${name}>\n${inner}\n</${name}>`;
|
|
||||||
};
|
|
||||||
|
|
||||||
if (data.message) {
|
|
||||||
// Ollama/compatible format
|
|
||||||
const thinking: string = data.message.thinking || '';
|
|
||||||
const contentText: string = data.message.content || data.response || '';
|
|
||||||
const textCombined = thinking && contentText ? `${thinking}\n\n${contentText}` : (thinking || contentText);
|
|
||||||
|
|
||||||
type WireToolCall = { function?: { name?: unknown; arguments?: unknown } };
|
|
||||||
const toolCallsRaw: unknown[] | undefined = data.message.tool_calls as unknown[] | undefined;
|
|
||||||
const toolCalls: Array<{ name: string; arguments: Record<string, unknown> }> = Array.isArray(toolCallsRaw)
|
|
||||||
? (toolCallsRaw
|
|
||||||
.map((tc: unknown) => {
|
|
||||||
const wire = tc as WireToolCall;
|
|
||||||
const fn = wire?.function;
|
|
||||||
if (!fn || typeof fn.name !== 'string') return null;
|
|
||||||
const args = fn.arguments;
|
|
||||||
if (args === null || typeof args !== 'object' || Array.isArray(args)) return null;
|
|
||||||
return { name: fn.name, arguments: args as Record<string, unknown> };
|
|
||||||
})
|
|
||||||
.filter((v): v is { name: string; arguments: Record<string, unknown> } => v !== null))
|
|
||||||
: [];
|
|
||||||
|
|
||||||
const xmlBlocks = toolCalls.length > 0
|
|
||||||
? toolCalls.map((tc) => `\n${functionCallToXml(tc.name, tc.arguments)}\n`).join('')
|
|
||||||
: '';
|
|
||||||
|
|
||||||
return {
|
|
||||||
content: `${textCombined}${xmlBlocks}`,
|
|
||||||
toolCalls: data.message.tool_calls || undefined,
|
|
||||||
finished: data.done === true || data.done_reason === 'stop'
|
|
||||||
};
|
|
||||||
} else {
|
|
||||||
// OpenAI format
|
|
||||||
const choice = data.choices?.[0];
|
|
||||||
const thinking: string = choice?.message?.thinking || '';
|
|
||||||
const contentText: string = choice?.message?.content || '';
|
|
||||||
const textCombined = thinking && contentText ? `${thinking}\n\n${contentText}` : (thinking || contentText);
|
|
||||||
|
|
||||||
type WireToolCall2 = { function?: { name?: unknown; arguments?: unknown } };
|
|
||||||
const toolCallsRaw: unknown[] | undefined = choice?.message?.tool_calls as unknown[] | undefined;
|
|
||||||
const toolCalls: Array<{ name: string; arguments: Record<string, unknown> }> = Array.isArray(toolCallsRaw)
|
|
||||||
? (toolCallsRaw
|
|
||||||
.map((tc: unknown) => {
|
|
||||||
const wire = tc as WireToolCall2;
|
|
||||||
const fn = wire?.function;
|
|
||||||
if (!fn || typeof fn.name !== 'string') return null;
|
|
||||||
const args = fn.arguments;
|
|
||||||
if (args === null || typeof args !== 'object' || Array.isArray(args)) return null;
|
|
||||||
return { name: fn.name, arguments: args as Record<string, unknown> };
|
|
||||||
})
|
|
||||||
.filter((v): v is { name: string; arguments: Record<string, unknown> } => v !== null))
|
|
||||||
: [];
|
|
||||||
|
|
||||||
const xmlBlocks = toolCalls.length > 0
|
|
||||||
? toolCalls.map((tc) => `\n${functionCallToXml(tc.name, tc.arguments)}\n`).join('')
|
|
||||||
: '';
|
|
||||||
|
|
||||||
return {
|
|
||||||
content: `${textCombined}${xmlBlocks}`,
|
|
||||||
toolCalls: choice?.message?.tool_calls || undefined,
|
|
||||||
finished: choice?.finish_reason === 'stop' || choice?.finish_reason === 'tool_calls'
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
+165
-295
@@ -1,5 +1,5 @@
|
|||||||
import React, { useState, useRef, useEffect, memo, useCallback } from 'react';
|
import React, { useState, useRef, useEffect, memo, useCallback } from 'react';
|
||||||
import { FaPlus, FaBan } from 'react-icons/fa';
|
import { FaPlus, FaBan, FaDownload } from 'react-icons/fa';
|
||||||
import { UserMessage, AssistantMessage } from './chat';
|
import { UserMessage, AssistantMessage } from './chat';
|
||||||
import { AgentCore } from '../agent/core/AgentCore';
|
import { AgentCore } from '../agent/core/AgentCore';
|
||||||
import { OpenAIProvider } from '../agent/llm/OpenAIProvider';
|
import { OpenAIProvider } from '../agent/llm/OpenAIProvider';
|
||||||
@@ -8,23 +8,22 @@ import { GeminiProvider } from '../agent/llm/GeminiProvider';
|
|||||||
import { LLMProvider } from '../agent/llm/LLMProvider';
|
import { LLMProvider } from '../agent/llm/LLMProvider';
|
||||||
import { ConfigManager } from '../core/config/ConfigManager';
|
import { ConfigManager } from '../core/config/ConfigManager';
|
||||||
import { useProjectStore } from '../stores/projectStore';
|
import { useProjectStore } from '../stores/projectStore';
|
||||||
import { XMLToolExecutor } from '../agent/core/XMLToolExecutor';
|
|
||||||
import { extractXMLFromString } from '../util/xmlUtil';
|
|
||||||
import { SystemPrompts } from '../agent/core/SystemPrompts';
|
import { SystemPrompts } from '../agent/core/SystemPrompts';
|
||||||
import { clearChatHistoryAndUI, registerClearChatUICallback } from '../util/chatUtil';
|
import { clearChatHistoryAndUI, registerClearChatUICallback } from '../util/chatUtil';
|
||||||
import { processUserMessage } from '../util/messageFilter/UserMessageFilter';
|
import { processUserMessage } from '../util/messageFilter/UserMessageFilter';
|
||||||
|
import { useStreamProcessor } from '../hooks/useStreamProcessor';
|
||||||
|
import { createMessage, addWelcomeMessage } from '../utils/chatMessageUtils';
|
||||||
|
import { extractActionableTools, executeAllTools } from '../utils/toolExecutionUtils';
|
||||||
|
import { formatLocalDateTime } from '../util/timeUtil';
|
||||||
|
import { downloadBlob, buildTimestampSuffix } from '../util/miscUtil';
|
||||||
|
import { wrapXmlBlocksInContent } from '../util/xmlUtil';
|
||||||
|
import KGDropdown from './common/KGDropdown';
|
||||||
|
|
||||||
|
import type { ChatMessage } from '../types/projectTypes';
|
||||||
|
|
||||||
// Module-level guard to avoid duplicate welcome in React StrictMode dev remounts
|
// Module-level guard to avoid duplicate welcome in React StrictMode dev remounts
|
||||||
let hasShownWelcomeOnceInRuntime = false;
|
let hasShownWelcomeOnceInRuntime = false;
|
||||||
|
|
||||||
interface ChatMessage {
|
|
||||||
id: string;
|
|
||||||
role: 'user' | 'assistant';
|
|
||||||
content: string;
|
|
||||||
isStreaming?: boolean;
|
|
||||||
tokenCount?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create the appropriate LLM provider based on configuration
|
* Create the appropriate LLM provider based on configuration
|
||||||
*/
|
*/
|
||||||
@@ -55,7 +54,6 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
|
|||||||
// Initialize with empty messages
|
// Initialize with empty messages
|
||||||
const [messages, setMessages] = useState<ChatMessage[]>([]);
|
const [messages, setMessages] = useState<ChatMessage[]>([]);
|
||||||
const [isProcessing, setIsProcessing] = useState(false);
|
const [isProcessing, setIsProcessing] = useState(false);
|
||||||
const [abortController, setAbortController] = useState<AbortController | null>(null);
|
|
||||||
const [lastUserMessage, setLastUserMessage] = useState<string>('');
|
const [lastUserMessage, setLastUserMessage] = useState<string>('');
|
||||||
|
|
||||||
// Tool execution state
|
// Tool execution state
|
||||||
@@ -66,10 +64,85 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
|
|||||||
// Track if this is the first message (for system prompt logging)
|
// Track if this is the first message (for system prompt logging)
|
||||||
const [isFirstMessage, setIsFirstMessage] = useState(true);
|
const [isFirstMessage, setIsFirstMessage] = useState(true);
|
||||||
|
|
||||||
const generateMessageId = (): string => {
|
// Export dropdown state and options
|
||||||
return `msg_${Date.now()}_${Math.random().toString(36).substring(2, 15)}`;
|
const [showExportDropdown, setShowExportDropdown] = useState(false);
|
||||||
|
const exportOptions = [
|
||||||
|
'Export conversation as JSON',
|
||||||
|
'Export conversation as Markdown'
|
||||||
|
];
|
||||||
|
|
||||||
|
const handleExportOptionSelect = (option: string) => {
|
||||||
|
if (option === 'Export conversation as JSON') {
|
||||||
|
try {
|
||||||
|
const messages = AgentCore.instance().getAgentState().getMessages();
|
||||||
|
const exportMessages = messages.map((m) => ({
|
||||||
|
id: m.id,
|
||||||
|
role: m.role,
|
||||||
|
content: m.content,
|
||||||
|
timestamp: formatLocalDateTime(new Date(m.timestamp))
|
||||||
|
}));
|
||||||
|
const json = JSON.stringify(exportMessages, null, 2);
|
||||||
|
const filename = `kgstudio-conversation-${buildTimestampSuffix()}.json`;
|
||||||
|
downloadBlob(json, 'application/json', filename);
|
||||||
|
} catch (err) {
|
||||||
|
console.error('Failed to export conversation as JSON:', err);
|
||||||
|
}
|
||||||
|
} else if (option === 'Export conversation as Markdown') {
|
||||||
|
(async () => {
|
||||||
|
try {
|
||||||
|
const messages = AgentCore.instance().getAgentState().getMessages();
|
||||||
|
const templateUrl = `${import.meta.env.BASE_URL}chat/export_conversation_template.md`;
|
||||||
|
const res = await fetch(templateUrl);
|
||||||
|
const template = await res.text();
|
||||||
|
|
||||||
|
const isAutomatedUserMessage = (content: string): boolean => {
|
||||||
|
return /^tool:\s.*\nsuccess:\s*(true|false)/i.test(content);
|
||||||
|
};
|
||||||
|
|
||||||
|
const sections = messages.map((m) => {
|
||||||
|
const isAutomaticUserMessage = isAutomatedUserMessage(m.content);
|
||||||
|
const roleLabel = m.role === 'assistant' ? 'Assistant' : (isAutomaticUserMessage ? 'User (Automatic)' : 'User');
|
||||||
|
const ts = formatLocalDateTime(new Date(m.timestamp));
|
||||||
|
const contentWithXml = isAutomaticUserMessage ? "```\n" + m.content + "\n```" : wrapXmlBlocksInContent(m.content);
|
||||||
|
return template
|
||||||
|
.replace('{role}', roleLabel)
|
||||||
|
.replace('{timestamp}', ts)
|
||||||
|
.replace('{content}', contentWithXml);
|
||||||
|
});
|
||||||
|
|
||||||
|
const markdown = sections.join('\n');
|
||||||
|
const filename = `kgstudio-conversation-${buildTimestampSuffix()}.md`;
|
||||||
|
downloadBlob(markdown, 'text/markdown', filename);
|
||||||
|
} catch (err) {
|
||||||
|
console.error('Failed to export conversation as Markdown:', err);
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
} else {
|
||||||
|
console.log('Chat export selected:', option);
|
||||||
|
}
|
||||||
|
setShowExportDropdown(false);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Message update callbacks for stream processor
|
||||||
|
const handleMessageUpdate = useCallback((messageId: string, updater: (msg: ChatMessage) => ChatMessage) => {
|
||||||
|
setMessages(prev => prev.map(msg => msg.id === messageId ? updater(msg) : msg));
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const handleMessageAdd = useCallback((message: ChatMessage) => {
|
||||||
|
setMessages(prev => [...prev, message]);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const handleProcessingChange = useCallback((processing: boolean) => {
|
||||||
|
setIsProcessing(processing);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
// Stream processor hook
|
||||||
|
const streamProcessor = useStreamProcessor({
|
||||||
|
onMessageUpdate: handleMessageUpdate,
|
||||||
|
onMessageAdd: handleMessageAdd,
|
||||||
|
onProcessingChange: handleProcessingChange
|
||||||
|
});
|
||||||
|
|
||||||
const clearChatUI = useCallback(async () => {
|
const clearChatUI = useCallback(async () => {
|
||||||
// Clear UI state
|
// Clear UI state
|
||||||
setMessages([]);
|
setMessages([]);
|
||||||
@@ -78,18 +151,9 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
|
|||||||
setIsFirstMessage(true);
|
setIsFirstMessage(true);
|
||||||
|
|
||||||
// Auto-show welcome message after clearing (like on app startup)
|
// Auto-show welcome message after clearing (like on app startup)
|
||||||
try {
|
const welcomeMessage = await addWelcomeMessage();
|
||||||
const result = await processUserMessage('/welcome');
|
if (welcomeMessage) {
|
||||||
if (result.pseudoAssistantResponse) {
|
setMessages([welcomeMessage]);
|
||||||
const pseudoId = generateMessageId();
|
|
||||||
setMessages(prev => [...prev, {
|
|
||||||
id: pseudoId,
|
|
||||||
role: 'assistant',
|
|
||||||
content: result.pseudoAssistantResponse!,
|
|
||||||
}]);
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
// ignore errors, just don't show welcome if it fails
|
|
||||||
}
|
}
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
@@ -117,29 +181,20 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
|
|||||||
|
|
||||||
// Auto-trigger welcome on first launch (guard against React StrictMode double-invoke only)
|
// Auto-trigger welcome on first launch (guard against React StrictMode double-invoke only)
|
||||||
(async () => {
|
(async () => {
|
||||||
try {
|
if (hasShownWelcomeOnceInRuntime) return;
|
||||||
if (hasShownWelcomeOnceInRuntime) return;
|
hasShownWelcomeOnceInRuntime = true;
|
||||||
hasShownWelcomeOnceInRuntime = true;
|
|
||||||
|
|
||||||
const result = await processUserMessage('/welcome');
|
const welcomeMessage = await addWelcomeMessage();
|
||||||
if (result.pseudoAssistantResponse) {
|
if (welcomeMessage) {
|
||||||
const pseudoId = generateMessageId();
|
setMessages([welcomeMessage]);
|
||||||
setMessages(prev => [...prev, {
|
|
||||||
id: pseudoId,
|
|
||||||
role: 'assistant',
|
|
||||||
content: result.pseudoAssistantResponse!,
|
|
||||||
}]);
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
// ignore
|
|
||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
}, [clearChatUI]);
|
}, [clearChatUI]);
|
||||||
|
|
||||||
const handleAbort = () => {
|
const handleAbort = () => {
|
||||||
if (abortController) {
|
const controller = streamProcessor.abortController;
|
||||||
abortController.abort();
|
if (controller) {
|
||||||
setAbortController(null);
|
controller.abort();
|
||||||
|
|
||||||
// Use AgentCore to clean up the data model and get the user message content
|
// Use AgentCore to clean up the data model and get the user message content
|
||||||
const agentCore = AgentCore.instance();
|
const agentCore = AgentCore.instance();
|
||||||
@@ -160,28 +215,9 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
|
|||||||
clearChatHistoryAndUI(setStatus);
|
clearChatHistoryAndUI(setStatus);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
const addToolResultMessage = (toolName: string, success: boolean, result: string) => {
|
|
||||||
const toolMsgId = generateMessageId();
|
|
||||||
const friendlyDisplay = `${success ? '✅' : '❌'} __**${toolName}**__ \n\n └── ${result}`;
|
|
||||||
|
|
||||||
setMessages(prev => [...prev, {
|
|
||||||
id: toolMsgId,
|
|
||||||
role: 'user',
|
|
||||||
content: friendlyDisplay
|
|
||||||
}]);
|
|
||||||
};
|
|
||||||
|
|
||||||
const executeToolsFromResponse = async (response: string): Promise<boolean> => {
|
const executeToolsFromResponse = async (response: string): Promise<boolean> => {
|
||||||
try {
|
try {
|
||||||
// Check if response contains XML tool invocations
|
const actionableBlocks = extractActionableTools(response);
|
||||||
const xmlBlocks = extractXMLFromString(response);
|
|
||||||
// Consider only actionable tools (exclude think/thinking)
|
|
||||||
const actionableBlocks = xmlBlocks.filter((block) => {
|
|
||||||
const match = block.match(/<([a-zA-Z_][a-zA-Z0-9_-]*)/);
|
|
||||||
const name = match ? match[1].toLowerCase() : '';
|
|
||||||
return name !== 'think' && name !== 'thinking';
|
|
||||||
});
|
|
||||||
|
|
||||||
if (actionableBlocks.length === 0) {
|
if (actionableBlocks.length === 0) {
|
||||||
// No actionable tools to execute, stop the loop
|
// No actionable tools to execute, stop the loop
|
||||||
@@ -194,46 +230,12 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
|
|||||||
setCurrentToolIndex(0);
|
setCurrentToolIndex(0);
|
||||||
|
|
||||||
const { setStatus } = useProjectStore.getState();
|
const { setStatus } = useProjectStore.getState();
|
||||||
setStatus(`Executing ${actionableBlocks.length} tool(s)...`);
|
|
||||||
|
// Execute all tools and get accumulated results
|
||||||
const executor = XMLToolExecutor.instance();
|
const accumulatedResults = await executeAllTools(actionableBlocks, {
|
||||||
let accumulatedResults = '';
|
onMessageAdd: handleMessageAdd,
|
||||||
|
onStatusUpdate: setStatus
|
||||||
// Execute tools sequentially with real-time updates
|
});
|
||||||
for (let i = 0; i < actionableBlocks.length; i++) {
|
|
||||||
setCurrentToolIndex(i + 1);
|
|
||||||
setStatus(`Executing tool ${i + 1} of ${actionableBlocks.length}...`);
|
|
||||||
|
|
||||||
// Determine tool name from XML block
|
|
||||||
const toolNameMatch = actionableBlocks[i].match(/<([a-zA-Z_][a-zA-Z0-9_-]*)/);
|
|
||||||
const toolName = toolNameMatch ? toolNameMatch[1] : 'unknown_tool';
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Execute single XML block
|
|
||||||
const results = await executor.executeXMLTools(actionableBlocks[i]);
|
|
||||||
const result = results[0]; // Single block should give single result
|
|
||||||
|
|
||||||
if (result) {
|
|
||||||
// Add friendly display message
|
|
||||||
addToolResultMessage(toolName, result.success, result.result);
|
|
||||||
|
|
||||||
// Accumulate formatted result for LLM (skip thinking tools)
|
|
||||||
if (toolName !== 'thinking' && toolName !== 'think') {
|
|
||||||
const formattedResult = `tool: ${toolName}\nsuccess: ${result.success}\nresult:\n${result.result}\n------------\n`;
|
|
||||||
accumulatedResults += formattedResult;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
// Handle individual tool error
|
|
||||||
addToolResultMessage(toolName, false, `Tool execution failed: ${error}`);
|
|
||||||
|
|
||||||
// Accumulate error result for LLM (skip thinking tools)
|
|
||||||
if (toolName !== 'thinking' && toolName !== 'think') {
|
|
||||||
const formattedResult = `tool: ${toolName}\nsuccess: false\nresult:\nTool execution failed: ${error}\n------------\n`;
|
|
||||||
accumulatedResults += formattedResult;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Store accumulated results
|
// Store accumulated results
|
||||||
setToolResults(accumulatedResults);
|
setToolResults(accumulatedResults);
|
||||||
@@ -264,90 +266,16 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
const sendToolResultsToLLM = async (toolResultsString: string): Promise<void> => {
|
const sendToolResultsToLLM = async (toolResultsString: string): Promise<void> => {
|
||||||
// Send tool results as hidden user input to LLM
|
// Process tool results through the stream processor
|
||||||
setIsProcessing(true);
|
const assistantResponse = await streamProcessor.processStream(toolResultsString, 'TOOL_RESULTS');
|
||||||
|
|
||||||
// Create abort controller for this request
|
// Check if the new response contains more tools
|
||||||
const controller = new AbortController();
|
const hasMoreTools = await executeToolsFromResponse(assistantResponse);
|
||||||
setAbortController(controller);
|
|
||||||
|
// If no more tools were found, set working flag to false
|
||||||
// Add streaming assistant message for the response
|
if (!hasMoreTools) {
|
||||||
const assistantMsgId = generateMessageId();
|
|
||||||
setMessages(prev => [...prev, {
|
|
||||||
id: assistantMsgId,
|
|
||||||
role: 'assistant',
|
|
||||||
content: '<span class="processing-wave">Processing...</span> 0 tokens received. click here to abort.',
|
|
||||||
isStreaming: true,
|
|
||||||
tokenCount: 0
|
|
||||||
}]);
|
|
||||||
|
|
||||||
try {
|
|
||||||
const agentCore = AgentCore.instance();
|
const agentCore = AgentCore.instance();
|
||||||
let assistantResponse = '';
|
agentCore.getAgentState().setIsWorkingOnTask(false);
|
||||||
let tokenCount = 0;
|
|
||||||
|
|
||||||
// Log the tool results being sent to LLM
|
|
||||||
console.log('------------ USER ------------');
|
|
||||||
console.log(toolResultsString);
|
|
||||||
console.log('------------------------------');
|
|
||||||
|
|
||||||
for await (const chunk of agentCore.processUserInput(toolResultsString)) {
|
|
||||||
// Check if request was aborted
|
|
||||||
if (controller.signal.aborted) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (chunk.type === 'text') {
|
|
||||||
assistantResponse += chunk.content;
|
|
||||||
tokenCount++;
|
|
||||||
|
|
||||||
// Update streaming message with token count and abort link
|
|
||||||
setMessages(prev => prev.map(msg =>
|
|
||||||
msg.id === assistantMsgId
|
|
||||||
? { ...msg, content: `<span class="processing-wave">Processing...</span> ${tokenCount} tokens received. click here to abort.`, tokenCount }
|
|
||||||
: msg
|
|
||||||
));
|
|
||||||
} else if (chunk.type === 'done') {
|
|
||||||
// Replace with final response
|
|
||||||
setMessages(prev => prev.map(msg =>
|
|
||||||
msg.id === assistantMsgId
|
|
||||||
? { ...msg, content: assistantResponse, isStreaming: false, tokenCount: undefined }
|
|
||||||
: msg
|
|
||||||
));
|
|
||||||
|
|
||||||
// Log the complete assistant response
|
|
||||||
console.log('------------ ASSISTANT ------------');
|
|
||||||
console.log(assistantResponse);
|
|
||||||
console.log('-----------------------------------');
|
|
||||||
|
|
||||||
// Check if the new response contains more tools
|
|
||||||
const hasMoreTools = await executeToolsFromResponse(assistantResponse);
|
|
||||||
|
|
||||||
// If no more tools were found, set working flag to false
|
|
||||||
if (!hasMoreTools) {
|
|
||||||
const agentCore = AgentCore.instance();
|
|
||||||
agentCore.getAgentState().setIsWorkingOnTask(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
if (error instanceof Error && error.name === 'AbortError') {
|
|
||||||
// Request was aborted, don't show error
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
console.error('Error processing tool results:', error);
|
|
||||||
// Update with error message
|
|
||||||
setMessages(prev => prev.map(msg =>
|
|
||||||
msg.id === assistantMsgId
|
|
||||||
? { ...msg, content: 'Error: Failed to process tool results', isStreaming: false, tokenCount: undefined }
|
|
||||||
: msg
|
|
||||||
));
|
|
||||||
} finally {
|
|
||||||
setAbortController(null);
|
|
||||||
setIsProcessing(false);
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -362,22 +290,14 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
|
|||||||
|
|
||||||
// Conditionally show the user message bubble
|
// Conditionally show the user message bubble
|
||||||
if (filterResult.displayUserMessage) {
|
if (filterResult.displayUserMessage) {
|
||||||
const userMsgId = generateMessageId();
|
const userMsgObject = createMessage('user', userMessage);
|
||||||
setMessages(prev => [...prev, {
|
handleMessageAdd(userMsgObject);
|
||||||
id: userMsgId,
|
|
||||||
role: 'user',
|
|
||||||
content: userMessage
|
|
||||||
}]);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// If we have a pseudo assistant response, show it immediately
|
// If we have a pseudo assistant response, show it immediately
|
||||||
if (filterResult.pseudoAssistantResponse) {
|
if (filterResult.pseudoAssistantResponse) {
|
||||||
const pseudoId = generateMessageId();
|
const pseudoMessage = createMessage('assistant', filterResult.pseudoAssistantResponse);
|
||||||
setMessages(prev => [...prev, {
|
handleMessageAdd(pseudoMessage);
|
||||||
id: pseudoId,
|
|
||||||
role: 'assistant',
|
|
||||||
content: filterResult.pseudoAssistantResponse!,
|
|
||||||
}]);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// If we shouldn't send anything to LLM, stop here
|
// If we shouldn't send anything to LLM, stop here
|
||||||
@@ -385,105 +305,33 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
setIsProcessing(true);
|
// Set working on task flag when user sends a message
|
||||||
|
const agentCore = AgentCore.instance();
|
||||||
|
agentCore.getAgentState().setIsWorkingOnTask(true);
|
||||||
|
|
||||||
// Create abort controller for this request
|
// Log system prompt only for first message or first message after clear
|
||||||
const controller = new AbortController();
|
if (isFirstMessage) {
|
||||||
setAbortController(controller);
|
try {
|
||||||
|
const systemPrompt = await SystemPrompts.getSystemPromptWithContext();
|
||||||
// Add streaming assistant message
|
console.log('------------ SYSTEM ------------');
|
||||||
const assistantMsgId = generateMessageId();
|
console.log(systemPrompt);
|
||||||
setMessages(prev => [...prev, {
|
console.log('--------------------------------');
|
||||||
id: assistantMsgId,
|
} catch (error) {
|
||||||
role: 'assistant',
|
console.error('Failed to log system prompt:', error);
|
||||||
content: '<span class="processing-wave">Processing...</span> 0 tokens received. click here to abort.',
|
|
||||||
isStreaming: true,
|
|
||||||
tokenCount: 0
|
|
||||||
}]);
|
|
||||||
|
|
||||||
try {
|
|
||||||
const agentCore = AgentCore.instance();
|
|
||||||
let assistantResponse = '';
|
|
||||||
let tokenCount = 0;
|
|
||||||
|
|
||||||
// Set working on task flag when user sends a message
|
|
||||||
agentCore.getAgentState().setIsWorkingOnTask(true);
|
|
||||||
|
|
||||||
// Log system prompt only for first message or first message after clear
|
|
||||||
if (isFirstMessage) {
|
|
||||||
try {
|
|
||||||
const systemPrompt = await SystemPrompts.getSystemPromptWithContext();
|
|
||||||
console.log('------------ SYSTEM ------------');
|
|
||||||
console.log(systemPrompt);
|
|
||||||
console.log('--------------------------------');
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Failed to log system prompt:', error);
|
|
||||||
}
|
|
||||||
// Mark that we've logged the system prompt for this conversation
|
|
||||||
setIsFirstMessage(false);
|
|
||||||
}
|
}
|
||||||
|
// Mark that we've logged the system prompt for this conversation
|
||||||
|
setIsFirstMessage(false);
|
||||||
|
}
|
||||||
|
|
||||||
// Log the final user message being sent to LLM
|
// Process user input through the stream processor
|
||||||
console.log('------------ USER ------------');
|
const assistantResponse = await streamProcessor.processStream(filterResult.finalMessageForLLM, 'USER');
|
||||||
console.log(filterResult.finalMessageForLLM);
|
|
||||||
console.log('------------------------------');
|
// Check if response contains tools to execute
|
||||||
|
const hasTools = await executeToolsFromResponse(assistantResponse);
|
||||||
for await (const chunk of agentCore.processUserInput(filterResult.finalMessageForLLM)) {
|
|
||||||
// Check if request was aborted
|
// If no tools were found, set working flag to false and return control to user
|
||||||
if (controller.signal.aborted) {
|
if (!hasTools) {
|
||||||
return;
|
agentCore.getAgentState().setIsWorkingOnTask(false);
|
||||||
}
|
|
||||||
|
|
||||||
if (chunk.type === 'text') {
|
|
||||||
assistantResponse += chunk.content;
|
|
||||||
tokenCount++;
|
|
||||||
|
|
||||||
// Update streaming message with token count and abort link
|
|
||||||
setMessages(prev => prev.map(msg =>
|
|
||||||
msg.id === assistantMsgId
|
|
||||||
? { ...msg, content: `<span class="processing-wave">Processing...</span> ${tokenCount} tokens received. click here to abort.`, tokenCount }
|
|
||||||
: msg
|
|
||||||
));
|
|
||||||
} else if (chunk.type === 'done') {
|
|
||||||
// Replace with final response
|
|
||||||
setMessages(prev => prev.map(msg =>
|
|
||||||
msg.id === assistantMsgId
|
|
||||||
? { ...msg, content: assistantResponse, isStreaming: false, tokenCount: undefined }
|
|
||||||
: msg
|
|
||||||
));
|
|
||||||
|
|
||||||
// Log the complete assistant response
|
|
||||||
console.log('------------ ASSISTANT ------------');
|
|
||||||
console.log(assistantResponse);
|
|
||||||
console.log('-----------------------------------');
|
|
||||||
|
|
||||||
// Check if response contains tools to execute
|
|
||||||
const hasTools = await executeToolsFromResponse(assistantResponse);
|
|
||||||
|
|
||||||
// If no tools were found, set working flag to false and return control to user
|
|
||||||
if (!hasTools) {
|
|
||||||
agentCore.getAgentState().setIsWorkingOnTask(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
if (error instanceof Error && error.name === 'AbortError') {
|
|
||||||
// Request was aborted, don't show error
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
console.error('Error processing message:', error);
|
|
||||||
// Update with error message
|
|
||||||
setMessages(prev => prev.map(msg =>
|
|
||||||
msg.id === assistantMsgId
|
|
||||||
? { ...msg, content: 'Error: Failed to process message', isStreaming: false, tokenCount: undefined }
|
|
||||||
: msg
|
|
||||||
));
|
|
||||||
} finally {
|
|
||||||
setAbortController(null);
|
|
||||||
setIsProcessing(false);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -538,7 +386,7 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
|
|||||||
}, [isProcessing, isExecutingTools]);
|
}, [isProcessing, isExecutingTools]);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="chatbox" style={{ display: isVisible ? 'flex' : 'none' }}>
|
<div className={`chatbox ${isVisible ? '' : 'is-hidden'}`}>
|
||||||
<div className="chatbox-header">
|
<div className="chatbox-header">
|
||||||
<h3>K.G.Studio Musician Assistant</h3>
|
<h3>K.G.Studio Musician Assistant</h3>
|
||||||
<div className="chatbox-actions">
|
<div className="chatbox-actions">
|
||||||
@@ -552,6 +400,28 @@ const ChatBox: React.FC<ChatBoxProps> = ({ isVisible }) => {
|
|||||||
<FaBan />
|
<FaBan />
|
||||||
</button>
|
</button>
|
||||||
)}
|
)}
|
||||||
|
<div className="chatbox-export-wrapper">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
title="Export"
|
||||||
|
onClick={() => setShowExportDropdown(!showExportDropdown)}
|
||||||
|
className="chatbox-action-btn chatbox-export-btn"
|
||||||
|
>
|
||||||
|
<FaDownload />
|
||||||
|
</button>
|
||||||
|
<div className="chatbox-export-dropdown-anchor">
|
||||||
|
<KGDropdown
|
||||||
|
options={exportOptions}
|
||||||
|
value={exportOptions[0]}
|
||||||
|
onChange={handleExportOptionSelect}
|
||||||
|
label="Export"
|
||||||
|
hideButton={true}
|
||||||
|
isOpen={showExportDropdown}
|
||||||
|
onToggle={setShowExportDropdown}
|
||||||
|
className="chatbox-export-dropdown"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
title="New Chat"
|
title="New Chat"
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { ConfigManager } from '../../../core/config/ConfigManager';
|
|||||||
|
|
||||||
const BehaviorSettings: React.FC = () => {
|
const BehaviorSettings: React.FC = () => {
|
||||||
const [chatboxDefaultOpen, setChatboxDefaultOpen] = useState<boolean>(true);
|
const [chatboxDefaultOpen, setChatboxDefaultOpen] = useState<boolean>(true);
|
||||||
|
const [enableAudioCapture, setEnableAudioCapture] = useState<boolean>(false);
|
||||||
|
|
||||||
const configManager = ConfigManager.instance();
|
const configManager = ConfigManager.instance();
|
||||||
|
|
||||||
@@ -14,6 +15,7 @@ const BehaviorSettings: React.FC = () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
setChatboxDefaultOpen((configManager.get('chatbox.default_open') as boolean) ?? true);
|
setChatboxDefaultOpen((configManager.get('chatbox.default_open') as boolean) ?? true);
|
||||||
|
setEnableAudioCapture((configManager.get('audio.enable_audio_capture_for_screen_sharing') as boolean) ?? false);
|
||||||
};
|
};
|
||||||
|
|
||||||
loadConfig();
|
loadConfig();
|
||||||
@@ -26,6 +28,12 @@ const BehaviorSettings: React.FC = () => {
|
|||||||
await configManager.set('chatbox.default_open', boolValue);
|
await configManager.set('chatbox.default_open', boolValue);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const handleEnableAudioCaptureChange = async (value: string) => {
|
||||||
|
const boolValue = value === 'yes';
|
||||||
|
setEnableAudioCapture(boolValue);
|
||||||
|
await configManager.set('audio.enable_audio_capture_for_screen_sharing', boolValue);
|
||||||
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="settings-section">
|
<div className="settings-section">
|
||||||
<div className="settings-section-header">
|
<div className="settings-section-header">
|
||||||
@@ -50,6 +58,29 @@ const BehaviorSettings: React.FC = () => {
|
|||||||
</select>
|
</select>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div className="settings-group">
|
||||||
|
<h4>Audio</h4>
|
||||||
|
|
||||||
|
<div className="settings-item">
|
||||||
|
<label className="settings-label">
|
||||||
|
Capture Audio for Screen Sharing
|
||||||
|
</label>
|
||||||
|
<select
|
||||||
|
className="settings-select"
|
||||||
|
value={enableAudioCapture ? 'yes' : 'no'}
|
||||||
|
onChange={(e) => handleEnableAudioCaptureChange(e.target.value)}
|
||||||
|
>
|
||||||
|
<option value="no">No</option>
|
||||||
|
<option value="yes">Yes</option>
|
||||||
|
</select>
|
||||||
|
<div className="settings-help" style={{ fontSize: '12px', color: '#888', marginTop: '4px' }}>
|
||||||
|
Restart KGStudio (refresh the page) to take effect. Enable this option when KGStudio's audio cannot be captured during screen sharing in video calls (e.g., Zoom, Teams). This creates an additional audio stream that screen capture applications can detect.
|
||||||
|
<br />
|
||||||
|
<b>It is important to make sure when screen sharing in Zoom, the "Share Sound" option is enabled.</b>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -0,0 +1,10 @@
|
|||||||
|
/**
|
||||||
|
* Constants related to LLM providers and API interactions
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Protocol markers and special character combinations used in LLM communications
|
||||||
|
export const LLM_PROTOCOL = {
|
||||||
|
SSE_DATA_PREFIX: 'data: ',
|
||||||
|
SSE_DONE_MARKER: '[DONE]',
|
||||||
|
SEGMENT_SEPARATOR: '\n\n'
|
||||||
|
} as const;
|
||||||
@@ -7,6 +7,7 @@ import * as Tone from 'tone';
|
|||||||
import { KGAudioBus } from './KGAudioBus';
|
import { KGAudioBus } from './KGAudioBus';
|
||||||
import type { InstrumentType } from '../track/KGMidiTrack';
|
import type { InstrumentType } from '../track/KGMidiTrack';
|
||||||
import { KGCore } from '../KGCore';
|
import { KGCore } from '../KGCore';
|
||||||
|
import { ConfigManager } from '../config/ConfigManager';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* KGAudioInterface - Audio engine interface for the DAW
|
* KGAudioInterface - Audio engine interface for the DAW
|
||||||
@@ -32,6 +33,10 @@ export class KGAudioInterface {
|
|||||||
// Master volume control
|
// Master volume control
|
||||||
private masterGain: Tone.Gain | null = null;
|
private masterGain: Tone.Gain | null = null;
|
||||||
|
|
||||||
|
// Audio capture for screen sharing
|
||||||
|
private captureDestination: MediaStreamAudioDestinationNode | null = null;
|
||||||
|
private captureStream: MediaStream | null = null;
|
||||||
|
|
||||||
// Private constructor to prevent direct instantiation
|
// Private constructor to prevent direct instantiation
|
||||||
private constructor() {
|
private constructor() {
|
||||||
console.log("KGAudioInterface initialized");
|
console.log("KGAudioInterface initialized");
|
||||||
@@ -66,6 +71,14 @@ export class KGAudioInterface {
|
|||||||
Tone.Transport.bpm.value = TIME_CONSTANTS.DEFAULT_BPM; // Default BPM
|
Tone.Transport.bpm.value = TIME_CONSTANTS.DEFAULT_BPM; // Default BPM
|
||||||
Tone.Transport.timeSignature = [TIME_CONSTANTS.DEFAULT_TIME_SIGNATURE.numerator, TIME_CONSTANTS.DEFAULT_TIME_SIGNATURE.denominator]; // Default time signature
|
Tone.Transport.timeSignature = [TIME_CONSTANTS.DEFAULT_TIME_SIGNATURE.numerator, TIME_CONSTANTS.DEFAULT_TIME_SIGNATURE.denominator]; // Default time signature
|
||||||
|
|
||||||
|
// Check config and setup audio capture if enabled
|
||||||
|
const configManager = ConfigManager.instance();
|
||||||
|
const enableCapture = configManager.get('audio.enable_audio_capture_for_screen_sharing') as boolean;
|
||||||
|
|
||||||
|
if (enableCapture) {
|
||||||
|
this.setupAudioCapture();
|
||||||
|
}
|
||||||
|
|
||||||
this.isInitialized = true;
|
this.isInitialized = true;
|
||||||
console.log("Audio engine initialized successfully");
|
console.log("Audio engine initialized successfully");
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -115,6 +128,12 @@ export class KGAudioInterface {
|
|||||||
this.masterGain = null;
|
this.masterGain = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Clean up capture resources
|
||||||
|
if (this.captureDestination) {
|
||||||
|
this.captureDestination = null;
|
||||||
|
this.captureStream = null;
|
||||||
|
}
|
||||||
|
|
||||||
this.isInitialized = false;
|
this.isInitialized = false;
|
||||||
this.isAudioContextStarted = false;
|
this.isAudioContextStarted = false;
|
||||||
|
|
||||||
@@ -602,8 +621,27 @@ export class KGAudioInterface {
|
|||||||
return Object.keys(FLUIDR3_INSTRUMENT_MAP) as InstrumentType[];
|
return Object.keys(FLUIDR3_INSTRUMENT_MAP) as InstrumentType[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public getCaptureStream(): MediaStream | null {
|
||||||
|
return this.captureStream;
|
||||||
|
}
|
||||||
|
|
||||||
// ===== PRIVATE UTILITY METHODS =====
|
// ===== PRIVATE UTILITY METHODS =====
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Setup audio capture for screen sharing
|
||||||
|
*/
|
||||||
|
private setupAudioCapture(): void {
|
||||||
|
if (this.masterGain && !this.captureDestination) {
|
||||||
|
this.captureDestination = Tone.getContext().createMediaStreamDestination();
|
||||||
|
this.captureStream = this.captureDestination.stream;
|
||||||
|
|
||||||
|
// Connect master gain to both speakers AND capture destination
|
||||||
|
this.masterGain.connect(this.captureDestination);
|
||||||
|
|
||||||
|
console.log('Audio capture enabled for screen sharing');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if any tracks are currently soloed
|
* Check if any tracks are currently soloed
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -60,6 +60,9 @@ interface AppConfig {
|
|||||||
chatbox: {
|
chatbox: {
|
||||||
default_open: boolean;
|
default_open: boolean;
|
||||||
};
|
};
|
||||||
|
audio: {
|
||||||
|
enable_audio_capture_for_screen_sharing: boolean;
|
||||||
|
};
|
||||||
templates: {
|
templates: {
|
||||||
custom_instructions: string;
|
custom_instructions: string;
|
||||||
};
|
};
|
||||||
@@ -205,6 +208,9 @@ export class ConfigManager {
|
|||||||
chatbox: {
|
chatbox: {
|
||||||
default_open: true
|
default_open: true
|
||||||
},
|
},
|
||||||
|
audio: {
|
||||||
|
enable_audio_capture_for_screen_sharing: false
|
||||||
|
},
|
||||||
templates: {
|
templates: {
|
||||||
custom_instructions: ''
|
custom_instructions: ''
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,119 @@
|
|||||||
|
import { useState, useCallback } from 'react';
|
||||||
|
import { AgentCore } from '../agent/core/AgentCore';
|
||||||
|
import { createStreamingMessage } from '../utils/chatMessageUtils';
|
||||||
|
import type { ChatMessage } from '../types/projectTypes';
|
||||||
|
|
||||||
|
interface StreamProcessorOptions {
|
||||||
|
onMessageUpdate: (messageId: string, updater: (msg: ChatMessage) => ChatMessage) => void;
|
||||||
|
onMessageAdd: (message: ChatMessage) => void;
|
||||||
|
onProcessingChange: (isProcessing: boolean) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface StreamProcessorResult {
|
||||||
|
processStream: (input: string, logPrefix?: string) => Promise<string>;
|
||||||
|
abortController: AbortController | null;
|
||||||
|
isProcessing: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const useStreamProcessor = (options: StreamProcessorOptions): StreamProcessorResult => {
|
||||||
|
const { onMessageUpdate, onMessageAdd, onProcessingChange } = options;
|
||||||
|
const [abortController, setAbortController] = useState<AbortController | null>(null);
|
||||||
|
const [isProcessing, setIsProcessing] = useState(false);
|
||||||
|
|
||||||
|
const processStream = useCallback(async (input: string, logPrefix: string = 'USER'): Promise<string> => {
|
||||||
|
setIsProcessing(true);
|
||||||
|
onProcessingChange(true);
|
||||||
|
|
||||||
|
// Create abort controller for this request
|
||||||
|
const controller = new AbortController();
|
||||||
|
setAbortController(controller);
|
||||||
|
|
||||||
|
// Add streaming assistant message
|
||||||
|
const streamingMessage = createStreamingMessage();
|
||||||
|
onMessageAdd(streamingMessage);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const agentCore = AgentCore.instance();
|
||||||
|
let assistantResponse = '';
|
||||||
|
let tokenCount = 0;
|
||||||
|
let streamCompleted = false;
|
||||||
|
|
||||||
|
// Log the input being sent to LLM
|
||||||
|
console.log(`------------ ${logPrefix} ------------`);
|
||||||
|
console.log(input);
|
||||||
|
console.log('------------------------------');
|
||||||
|
|
||||||
|
for await (const chunk of agentCore.processUserInput(input)) {
|
||||||
|
// Check if request was aborted
|
||||||
|
if (controller.signal.aborted) {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
|
||||||
|
if (chunk.type === 'text') {
|
||||||
|
assistantResponse += chunk.content;
|
||||||
|
tokenCount++;
|
||||||
|
|
||||||
|
// Update streaming message with token count and abort link
|
||||||
|
onMessageUpdate(streamingMessage.id, (msg) => ({
|
||||||
|
...msg,
|
||||||
|
content: `<span class="processing-wave">Processing...</span> ${tokenCount} tokens received. click here to abort.`,
|
||||||
|
tokenCount
|
||||||
|
}));
|
||||||
|
} else if (chunk.type === 'done') {
|
||||||
|
streamCompleted = true;
|
||||||
|
// Replace with final response
|
||||||
|
onMessageUpdate(streamingMessage.id, (msg) => ({
|
||||||
|
...msg,
|
||||||
|
content: assistantResponse,
|
||||||
|
isStreaming: false,
|
||||||
|
tokenCount: undefined
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Log the complete assistant response
|
||||||
|
console.log('------------ ASSISTANT ------------');
|
||||||
|
console.log(assistantResponse);
|
||||||
|
console.log('-----------------------------------');
|
||||||
|
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If stream didn't complete normally, finalize the message
|
||||||
|
if (!streamCompleted && !controller.signal.aborted) {
|
||||||
|
onMessageUpdate(streamingMessage.id, (msg) => ({
|
||||||
|
...msg,
|
||||||
|
content: assistantResponse || 'Stream was interrupted unexpectedly',
|
||||||
|
isStreaming: false,
|
||||||
|
tokenCount: undefined
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
return assistantResponse;
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof Error && error.name === 'AbortError') {
|
||||||
|
// Request was aborted, don't show error
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
|
||||||
|
console.error('Error processing stream:', error);
|
||||||
|
// Update with error message
|
||||||
|
onMessageUpdate(streamingMessage.id, (msg) => ({
|
||||||
|
...msg,
|
||||||
|
content: 'Error: Failed to process message',
|
||||||
|
isStreaming: false,
|
||||||
|
tokenCount: undefined
|
||||||
|
}));
|
||||||
|
return '';
|
||||||
|
} finally {
|
||||||
|
setAbortController(null);
|
||||||
|
setIsProcessing(false);
|
||||||
|
onProcessingChange(false);
|
||||||
|
}
|
||||||
|
}, [onMessageUpdate, onMessageAdd, onProcessingChange]);
|
||||||
|
|
||||||
|
return {
|
||||||
|
processStream,
|
||||||
|
abortController,
|
||||||
|
isProcessing
|
||||||
|
};
|
||||||
|
};
|
||||||
@@ -5,6 +5,14 @@ export interface TimeSignature {
|
|||||||
denominator: number;
|
denominator: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface ChatMessage {
|
||||||
|
id: string;
|
||||||
|
role: 'user' | 'assistant';
|
||||||
|
content: string;
|
||||||
|
isStreaming?: boolean;
|
||||||
|
tokenCount?: number;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A reusable class-transformer decorator to apply a default value during deserialization.
|
* A reusable class-transformer decorator to apply a default value during deserialization.
|
||||||
* @param defaultValue The default value to apply if the field is undefined.
|
* @param defaultValue The default value to apply if the field is undefined.
|
||||||
|
|||||||
@@ -82,9 +82,36 @@ export async function processUserMessage(originalMessage: string): Promise<UserM
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case '/help': {
|
||||||
|
try {
|
||||||
|
const url = `${import.meta.env.BASE_URL}chat/help.md`;
|
||||||
|
const resp = await fetch(url);
|
||||||
|
if (!resp.ok) {
|
||||||
|
throw new Error(`Failed to fetch ${url}: ${resp.status}`);
|
||||||
|
}
|
||||||
|
const md = await resp.text();
|
||||||
|
return {
|
||||||
|
displayUserMessage: false,
|
||||||
|
sendToLLM: false,
|
||||||
|
finalMessageForLLM: null,
|
||||||
|
pseudoAssistantResponse: md,
|
||||||
|
metadata: { command: 'help' }
|
||||||
|
};
|
||||||
|
} catch (err) {
|
||||||
|
const fallback = 'Help is currently unavailable.';
|
||||||
|
return {
|
||||||
|
displayUserMessage: false,
|
||||||
|
sendToLLM: false,
|
||||||
|
finalMessageForLLM: null,
|
||||||
|
pseudoAssistantResponse: fallback,
|
||||||
|
metadata: { command: 'help', error: String(err) }
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
default: {
|
default: {
|
||||||
const { setStatus } = useProjectStore.getState();
|
const { setStatus } = useProjectStore.getState();
|
||||||
const help = 'Available commands: /clear, /welcome';
|
const help = 'Available commands: /clear, /welcome, /help';
|
||||||
setStatus(`Unknown command: ${command}. ${help}`);
|
setStatus(`Unknown command: ${command}. ${help}`);
|
||||||
return {
|
return {
|
||||||
displayUserMessage: false,
|
displayUserMessage: false,
|
||||||
|
|||||||
+24
-1
@@ -66,4 +66,27 @@ export const generateNewRegionName = (trackId: string): string => {
|
|||||||
}
|
}
|
||||||
i++;
|
i++;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/** Create and trigger a download for a text/blob payload */
|
||||||
|
export function downloadBlob(content: BlobPart, mimeType: string, filename: string): void {
|
||||||
|
const blob = new Blob([content], { type: mimeType });
|
||||||
|
const url = URL.createObjectURL(blob);
|
||||||
|
const link = document.createElement('a');
|
||||||
|
link.href = url;
|
||||||
|
link.download = filename;
|
||||||
|
document.body.appendChild(link);
|
||||||
|
link.click();
|
||||||
|
document.body.removeChild(link);
|
||||||
|
URL.revokeObjectURL(url);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Pad number with leading zero to 2 digits */
|
||||||
|
export function pad2(n: number): string {
|
||||||
|
return n.toString().padStart(2, '0');
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Build timestamped filename suffix YYYY-MM-DD-hh-mm-ss */
|
||||||
|
export function buildTimestampSuffix(date: Date = new Date()): string {
|
||||||
|
return `${date.getFullYear()}-${pad2(date.getMonth() + 1)}-${pad2(date.getDate())}-${pad2(date.getHours())}-${pad2(date.getMinutes())}-${pad2(date.getSeconds())}`;
|
||||||
|
}
|
||||||
+19
-1
@@ -87,4 +87,22 @@ export function beatsToTimeString(
|
|||||||
const mmm = milliseconds.toString().padStart(3, '0');
|
const mmm = milliseconds.toString().padStart(3, '0');
|
||||||
|
|
||||||
return `${BBB}:${B} | ${mm}:${ss}:${mmm}`;
|
return `${BBB}:${B} | ${mm}:${ss}:${mmm}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Format a Date object into a human-readable local datetime string with timezone.
|
||||||
|
* Example: 2025/08/21, 19:57:11 GMT-7
|
||||||
|
*/
|
||||||
|
export function formatLocalDateTime(date: Date): string {
|
||||||
|
const options: Intl.DateTimeFormatOptions = {
|
||||||
|
year: 'numeric',
|
||||||
|
month: '2-digit',
|
||||||
|
day: '2-digit',
|
||||||
|
hour: '2-digit',
|
||||||
|
minute: '2-digit',
|
||||||
|
second: '2-digit',
|
||||||
|
hour12: false,
|
||||||
|
timeZoneName: 'short'
|
||||||
|
};
|
||||||
|
return date.toLocaleString(undefined, options);
|
||||||
|
}
|
||||||
+18
-1
@@ -49,4 +49,21 @@ export function extractXMLFromString(input: string): string[] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return matches;
|
return matches;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wrap XML blocks in content with fenced code blocks
|
||||||
|
* @param content - The input string that may contain XML blocks
|
||||||
|
* @returns The input string with XML blocks wrapped in fenced code blocks
|
||||||
|
*/
|
||||||
|
export const wrapXmlBlocksInContent = (content: string): string => {
|
||||||
|
if (!content) return content;
|
||||||
|
const xmlBlocks = extractXMLFromString(content);
|
||||||
|
if (!xmlBlocks || xmlBlocks.length === 0) return content;
|
||||||
|
let result = content;
|
||||||
|
for (const block of xmlBlocks) {
|
||||||
|
const fenced = '```xml\n' + block + '\n```';
|
||||||
|
result = result.split(block).join(fenced);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
};
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
import { processUserMessage } from '../util/messageFilter/UserMessageFilter';
|
||||||
|
import type { ChatMessage } from '../types/projectTypes';
|
||||||
|
|
||||||
|
export const generateMessageId = (): string => {
|
||||||
|
return `msg_${Date.now()}_${Math.random().toString(36).substring(2, 15)}`;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const createMessage = (role: 'user' | 'assistant', content: string): ChatMessage => {
|
||||||
|
return {
|
||||||
|
id: generateMessageId(),
|
||||||
|
role,
|
||||||
|
content,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export const createStreamingMessage = (content: string = '<span class="processing-wave">Processing...</span> 0 tokens received. click here to abort.'): ChatMessage => {
|
||||||
|
return {
|
||||||
|
id: generateMessageId(),
|
||||||
|
role: 'assistant',
|
||||||
|
content,
|
||||||
|
isStreaming: true,
|
||||||
|
tokenCount: 0,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export const createToolResultMessage = (toolName: string, success: boolean, result: string): ChatMessage => {
|
||||||
|
const friendlyDisplay = `${success ? '✅' : '❌'} __**${toolName}**__ \n\n └── ${result}`;
|
||||||
|
return createMessage('user', friendlyDisplay);
|
||||||
|
};
|
||||||
|
|
||||||
|
export const addWelcomeMessage = async (): Promise<ChatMessage | null> => {
|
||||||
|
try {
|
||||||
|
const result = await processUserMessage('/welcome');
|
||||||
|
if (result.pseudoAssistantResponse) {
|
||||||
|
return createMessage('assistant', result.pseudoAssistantResponse);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
} catch {
|
||||||
|
// ignore errors, just don't show welcome if it fails
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
};
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
import { XMLToolExecutor } from '../agent/core/XMLToolExecutor';
|
||||||
|
import { extractXMLFromString } from '../util/xmlUtil';
|
||||||
|
import { createToolResultMessage } from './chatMessageUtils';
|
||||||
|
import type { ChatMessage } from '../types/projectTypes';
|
||||||
|
|
||||||
|
interface ToolExecutionResult {
|
||||||
|
success: boolean;
|
||||||
|
result: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ToolExecutionOptions {
|
||||||
|
onMessageAdd: (message: ChatMessage) => void;
|
||||||
|
onStatusUpdate: (status: string) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const extractActionableTools = (response: string): string[] => {
|
||||||
|
const xmlBlocks = extractXMLFromString(response);
|
||||||
|
// Consider only actionable tools (exclude think/thinking)
|
||||||
|
return xmlBlocks.filter((block) => {
|
||||||
|
const match = block.match(/<([a-zA-Z_][a-zA-Z0-9_-]*)/);
|
||||||
|
const name = match ? match[1].toLowerCase() : '';
|
||||||
|
return name !== 'think' && name !== 'thinking';
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
export const extractToolName = (xmlBlock: string): string => {
|
||||||
|
const toolNameMatch = xmlBlock.match(/<([a-zA-Z_][a-zA-Z0-9_-]*)/);
|
||||||
|
return toolNameMatch ? toolNameMatch[1] : 'unknown_tool';
|
||||||
|
};
|
||||||
|
|
||||||
|
export const executeSingleTool = async (
|
||||||
|
xmlBlock: string,
|
||||||
|
toolName: string,
|
||||||
|
options: ToolExecutionOptions
|
||||||
|
): Promise<ToolExecutionResult> => {
|
||||||
|
const { onMessageAdd } = options;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const executor = XMLToolExecutor.instance();
|
||||||
|
const results = await executor.executeXMLTools(xmlBlock);
|
||||||
|
const result = results[0]; // Single block should give single result
|
||||||
|
|
||||||
|
if (result) {
|
||||||
|
// Add friendly display message
|
||||||
|
const toolMessage = createToolResultMessage(toolName, result.success, result.result);
|
||||||
|
onMessageAdd(toolMessage);
|
||||||
|
|
||||||
|
return {
|
||||||
|
success: result.success,
|
||||||
|
result: result.result
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
result: 'No result returned from tool execution'
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
// Handle individual tool error
|
||||||
|
const errorMessage = `Tool execution failed: ${error}`;
|
||||||
|
const toolMessage = createToolResultMessage(toolName, false, errorMessage);
|
||||||
|
onMessageAdd(toolMessage);
|
||||||
|
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
result: errorMessage
|
||||||
|
};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export const formatToolResultForLLM = (toolName: string, result: ToolExecutionResult): string => {
|
||||||
|
// Skip thinking tools
|
||||||
|
if (toolName === 'thinking' || toolName === 'think') {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
|
||||||
|
return `tool: ${toolName}\nsuccess: ${result.success}\nresult:\n${result.result}\n------------\n`;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const executeAllTools = async (
|
||||||
|
actionableBlocks: string[],
|
||||||
|
options: ToolExecutionOptions
|
||||||
|
): Promise<string> => {
|
||||||
|
const { onStatusUpdate } = options;
|
||||||
|
|
||||||
|
onStatusUpdate(`Executing ${actionableBlocks.length} tool(s)...`);
|
||||||
|
|
||||||
|
let accumulatedResults = '';
|
||||||
|
|
||||||
|
// Execute tools sequentially with real-time updates
|
||||||
|
for (let i = 0; i < actionableBlocks.length; i++) {
|
||||||
|
onStatusUpdate(`Executing tool ${i + 1} of ${actionableBlocks.length}...`);
|
||||||
|
|
||||||
|
const toolName = extractToolName(actionableBlocks[i]);
|
||||||
|
const result = await executeSingleTool(actionableBlocks[i], toolName, options);
|
||||||
|
|
||||||
|
// Accumulate formatted result for LLM
|
||||||
|
accumulatedResults += formatToolResultForLLM(toolName, result);
|
||||||
|
}
|
||||||
|
|
||||||
|
return accumulatedResults;
|
||||||
|
};
|
||||||
Reference in New Issue
Block a user