rewrote welcome_new article; updated readme file; introduced /help command.
This commit is contained in:
Xiaohan-Tian
@@ -14,6 +14,8 @@ K.G.Studio is a lightweight, modern DAW that runs entirely in the browser with *
|
||||
|
||||
This project investigates how AI-human collaboration can enhance creative music-making, from intelligent harmony suggestions to automated editing tasks. As an experimental platform, expect frequent changes, evolving features, and occasional instability as we push the boundaries of what's possible in AI-assisted music production.
|
||||
|
||||
### Start using the app online: [K.G.Studio (kgaudiolab.github.io/kgstudio)](https://kgaudiolab.github.io/kgstudio)
|
||||
|
||||
## Demo Videos
|
||||
|
||||
<div align="center">
|
||||
@@ -35,6 +37,30 @@ This project investigates how AI-human collaboration can enhance creative music-
|
||||
</table>
|
||||
</div>
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Setting up K.G.Studio Musician Assistant
|
||||
- [Click here to get a free OpenRouter API Key](https://openrouter.ai/keys) (you may need an OpenRouter account).
|
||||
- In **Settings ⚙️ → General → LLM Provider**, select **OpenAI Compatible**.
|
||||
- In **OpenAI Compatible Server → Key**, paste your key. (Note: on non‑localhost, your key isn’t persisted; you’ll re‑enter it after refresh the page. Keep it safe.)
|
||||
- In **OpenAI Compatible Server → Model**, enter `qwen/qwen3-30b-a3b:free`. (Note: non‑free models may require billing; some model providers may retain your data, check their privacy policies; this project is not affiliated with OpenRouter or any model provider.)
|
||||
- In **OpenAI Compatible Server → Base URL**, enter `https://openrouter.ai/api/v1/chat/completions`.
|
||||
|
||||
(Alternatively, you can use the official OpenAI API, other OpenAI‑compatible services, or your own hosted LLM server. e.g., Ollama, vLLM)
|
||||
|
||||
### Basic DAW operations
|
||||
- Double‑click (or hold Ctrl/Cmd and click) on a track to create a region.
|
||||
- Drag region edges to resize; drag the body to move.
|
||||
- Click the small pencil at a region’s top‑left to open the piano roll.
|
||||
- In the piano roll, double‑click (or Ctrl/Cmd+click) to create a note.
|
||||
- Click to select; Shift+click for multi‑select; drag to box‑select.
|
||||
- Drag note edges to resize; drag the note body to move selected notes.
|
||||
- Use Snapping in the piano roll toolbar (top‑right) to quantize to grid.
|
||||
|
||||
### More details
|
||||
|
||||
You can find the detailed user guide [here](./docs/USER_GUIDE.md).
|
||||
|
||||
### Highlights
|
||||
- **K.G.Studio Musician Assistant**: Chat with the LLM‑powered K.G.Studio Musician Assistant AI Agent; it can automatically execute tools to make music edits.
|
||||
- **Multiple LLM providers**: OpenAI, Claude (via OpenRouter), Gemini (via OpenRouter), or OpenAI‑compatible (e.g., Ollama, OpenRouter).
|
||||
@@ -50,8 +76,6 @@ For a deeper technical overview, see `res/overview.md`.
|
||||
|
||||
## Getting Started
|
||||
|
||||
### Start using the app online: [K.G.Studio (kgaudiolab.github.io/kgstudio)](https://kgaudiolab.github.io/kgstudio)
|
||||
|
||||
### Or clone and run locally:
|
||||
```bash
|
||||
# Make sure you have Node.js >= 20.19.3 installed
|
||||
@@ -94,16 +118,33 @@ K.G.Studio loads defaults from `./public/config.json` (with an internal fallback
|
||||
|
||||
You can find the detailed user guide [here](./docs/USER_GUIDE.md).
|
||||
|
||||
1. Start the app and a default project will load. A default “Melody” MIDI track is ensured on load.
|
||||
2. Add tracks, rename and reorder them. Click or drag in the track grid to create and arrange regions.
|
||||
3. Add a region by double‑clicking or holding Ctrl/Cmd and clicking. Drag the edges to resize a region; drag the center to move it. Use Ctrl/Cmd+C and Ctrl/Cmd+V to copy/paste regions.
|
||||
4. Open a MIDI region to edit notes in the Piano Roll window (click the pencil icon on the region’s top‑left corner, or click the Piano icon on the toolbar).
|
||||
5. Double‑click or hold Ctrl/Cmd and click to create a note. Drag the edges to change its length, or drag the center to move it. Use box selection or hold Shift to multi‑select.
|
||||
6. Quantize note length/position using the Quantize dropdowns at the top‑right of the piano roll.
|
||||
7. Change instruments via the Instrument Selection panel; samples are downloaded on demand.
|
||||
8. Save projects to your browser’s IndexedDB and load by name from the toolbar. Export your project as a MIDI file, or import a MIDI file into your project.
|
||||
- Tracks
|
||||
- Add, rename, and reorder tracks from the track info panel.
|
||||
- Change instrument using the instrument button (piano icon); adjust Solo (S), Mute (M), and Volume.
|
||||
- Delete a track from the track’s settings menu (button to the right of the instrument).
|
||||
|
||||
## Keyboard Shortcuts (defaults)
|
||||
- Regions
|
||||
- Create region: with the Pointer tool, double‑click; or hold Ctrl/Cmd and click. With the Pencil tool, single‑click.
|
||||
- Move/resize: drag the body to move; drag edges to resize.
|
||||
- Open Piano Roll via the small pencil at a region’s top‑left.
|
||||
|
||||
- Piano Roll (MIDI notes)
|
||||
- Tools: Select vs Pencil.
|
||||
- Create notes: double‑click or Ctrl/Cmd+click (Select); single‑click (Pencil).
|
||||
- Select notes: click; Shift+click for multi‑select; drag to box‑select.
|
||||
- Move/resize: drag note body to move selected notes; drag edges to resize.
|
||||
- Close the piano roll with X or ESC.
|
||||
|
||||
- Snapping and Quantize
|
||||
- Set snapping from the NO SNAP menu (top‑right).
|
||||
- Quantize timing with Qua. Pos. (start) and Qua. Len. (length).
|
||||
|
||||
- Playback & Playhead
|
||||
- Back to beginning; Play/Pause from the toolbar.
|
||||
- Set the playhead by clicking bar numbers in the main grid; in Piano Roll, click the header timeline (respects snapping).
|
||||
- Change BPM, time signature, and key signature via the toolbar readouts.
|
||||
|
||||
## Keyboard Shortcuts
|
||||
|
||||
- Main
|
||||
- Play/Pause: Space
|
||||
@@ -120,11 +161,11 @@ You can find the detailed user guide [here](./docs/USER_GUIDE.md).
|
||||
|
||||
## AI Assistant
|
||||
|
||||
### Quick Start
|
||||
### Using K.G.Studio Musician Assistant
|
||||
|
||||
- First, select an LLM provider in **Settings ⚙️ → General → LLM Provider**. If you are using OpenAI (e.g., GPT‑4o), get your API key from [**OpenAI**](https://platform.openai.com/account/api-keys) and paste it in **OpenAI → Key**.
|
||||
- Make sure you have followed the previous section to set up the LLM provider.
|
||||
- You can find the K.G.Studio Musician Assistant chatbox on the right. If you don't see it, you can click the Chat 🗨️ button on the toolbar.
|
||||
- Type your message in the chatbox; press Enter to send, Shift+Enter to insert a new line.
|
||||
- Select the region you want the assistant to work on, type your prompt in the chatbox; press Enter to send, Shift+Enter to insert a new line.
|
||||
- The agent will automatically process your request and invoke tools to make modifications scoped to the selected region. It may take one or more turns to complete a task.
|
||||
- Note that the AI could make mistakes, so you should always check the result and make adjustments if necessary. You can also use undo/redo to revert the changes.
|
||||
- Click the "+" button or `/clear` command to clear the chat history.
|
||||
@@ -153,11 +194,9 @@ OpenRouter is a platform that provides unified access to a wide range of languag
|
||||
4. Browse available models on the [**OpenRouter Models Page**](https://openrouter.ai/models). Use the "Prompt Pricing" filter to identify free models.
|
||||
**Note:** Each model provider may have different data retention and privacy policies. Please review these policies before use.
|
||||
5. Enter your chosen model name in **OpenAI Compatible Server → Model**. Recommended model series include:
|
||||
- `Google: gemini-2.5-pro`
|
||||
- `Anthropic: claude-4-sonnet`
|
||||
- `DeepSeek: deepseek-r1` (free: `deepseek/deepseek-r1-0528:free`)
|
||||
- `DeepSeek: deepseek-v3` (free: `deepseek/deepseek-chat-v3-0324:free`)
|
||||
- `Qwen: qwen3-235b-a22b` (free: `qwen/qwen3-235b-a22b:free`)
|
||||
- `Anthropic: claude-4-sonnet` (`anthropic/claude-sonnet-4`: [Link](https://openrouter.ai/anthropic/claude-sonnet-4))
|
||||
- `Qwen: qwen3-30b-a3b` (FREE MODEL: `qwen/qwen3-30b-a3b:free`: [Link](https://openrouter.ai/qwen/qwen3-30b-a3b:free))
|
||||
- `Qwen: qwen3-235b-a22b` (FREE MODEL: `qwen/qwen3-235b-a22b:free`: [Link](https://openrouter.ai/qwen/qwen3-235b-a22b:free))
|
||||
6. Input the base URL `https://openrouter.ai/api/v1/chat/completions` **OpenAI Compatible Server → Base URL**.
|
||||
|
||||
### About the agent and LLM providers
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
## Help
|
||||
|
||||
Welcome to **K.G.Studio Musician Assistant**—your AI-powered agent for music composition and arrangement. Harnessing advanced language models, I am here to help you create, arrange, and refine your musical ideas with ease and intelligence.
|
||||
|
||||
To get started, you will need to configure an LLM (Large Language Model) Provider in the application settings. Please follow the instructions below to ensure proper setup:
|
||||
|
||||
---
|
||||
|
||||
### Configuring Your LLM Provider
|
||||
|
||||
Navigate to **Settings ⚙️ → General → LLM Provider**. Depending on your chosen provider, you will need to supply the appropriate API Key and, if applicable, a custom base URL (for non-official OpenAI-compatible services such as Ollama, OpenRouter, etc.).
|
||||
|
||||
---
|
||||
|
||||
### Using the OpenAI GPT Model Series
|
||||
|
||||
1. Obtain an OpenAI API Key from [**OpenAI**](https://platform.openai.com/account/api-keys). You may need to create an account and add a payment method to generate an API Key.
|
||||
2. In **Settings ⚙️ → General → LLM Provider**, select **OpenAI** as your provider.
|
||||
3. Enter your API Key in **OpenAI → Key**.
|
||||
4. Select your preferred model from the **OpenAI → Model** dropdown. For optimal performance, we recommend `gpt-5` or `gpt-4o`.
|
||||
5. Optionally, choose whether to enable Flex Mode in **OpenAI → Flex Mode**. Flex Mode offers discounted pricing, but may result in slower response times.
|
||||
|
||||
---
|
||||
|
||||
### Using OpenRouter
|
||||
|
||||
OpenRouter is a platform that provides unified access to a wide range of language models—including free options—from various providers. This makes it easy to experiment and find the model that best suits your needs.
|
||||
|
||||
1. Obtain an API Key from [**OpenRouter**](https://openrouter.ai/keys). Registration is required; for paid models, a payment method may also be necessary.
|
||||
2. In **Settings ⚙️ → General → LLM Provider**, select **OpenAI Compatible** as your provider.
|
||||
3. Enter your API Key in **OpenAI Compatible Server → Key**.
|
||||
4. Browse available models on the [**OpenRouter Models Page**](https://openrouter.ai/models). Use the "Prompt Pricing" filter to identify free models.
|
||||
**Note:** Each model provider may have different data retention and privacy policies. Please review these policies before use.
|
||||
5. Enter your chosen model name in **OpenAI Compatible Server → Model**. Recommended model series include:
|
||||
- `Anthropic: claude-4-sonnet` (`anthropic/claude-sonnet-4`: [Link](https://openrouter.ai/anthropic/claude-sonnet-4))
|
||||
- `Qwen: qwen3-30b-a3b` (FREE MODEL: `qwen/qwen3-30b-a3b:free`: [Link](https://openrouter.ai/qwen/qwen3-30b-a3b:free))
|
||||
- `Qwen: qwen3-235b-a22b` (FREE MODEL: `qwen/qwen3-235b-a22b:free`: [Link](https://openrouter.ai/qwen/qwen3-235b-a22b:free))
|
||||
6. Input the base URL `https://openrouter.ai/api/v1/chat/completions` of the OpenAI Compatible Server in **OpenAI Compatible Server → Base URL**.
|
||||
|
||||
### Basic DAW Operations
|
||||
|
||||
- Tracks
|
||||
- Add, rename, and reorder tracks from the track info panel.
|
||||
- Change instrument using the instrument button (piano icon); adjust Solo (S), Mute (M), and Volume.
|
||||
- Delete a track from the track’s settings menu (button to the right of the instrument).
|
||||
|
||||
- Regions
|
||||
- Create region: with the Pointer tool, double‑click; or hold Ctrl/Cmd and click. With the Pencil tool, single‑click.
|
||||
- Move/resize: drag the body to move; drag edges to resize.
|
||||
- Open Piano Roll via the small pencil at a region’s top‑left.
|
||||
|
||||
- Piano Roll (MIDI notes)
|
||||
- Tools: Select vs Pencil.
|
||||
- Create notes: double‑click or Ctrl/Cmd+click (Select); single‑click (Pencil).
|
||||
- Select notes: click; Shift+click for multi‑select; drag to box‑select.
|
||||
- Move/resize: drag note body to move selected notes; drag edges to resize.
|
||||
- Close the piano roll with X or ESC.
|
||||
|
||||
- Snapping and Quantize
|
||||
- Set snapping from the NO SNAP menu (top‑right).
|
||||
- Quantize timing with Qua. Pos. (start) and Qua. Len. (length).
|
||||
|
||||
- Playback & Playhead
|
||||
- Back to beginning; Play/Pause from the toolbar.
|
||||
- Set the playhead by clicking bar numbers in the main grid; in Piano Roll, click the header timeline (respects snapping).
|
||||
- Change BPM, time signature, and key signature via the toolbar readouts.
|
||||
|
||||
---
|
||||
|
||||
### Note
|
||||
|
||||
Due to security reasons, if you are using K.G.Studio from a non-local hosted environment, we won't persist your API Key in your browser's IndexedDB in order to prevent potential XSS attack, you will need to input your API Key every time you start K.G.Studio.
|
||||
|
||||
### Disclaimer
|
||||
|
||||
K.G.Studio does not provide or host any of the models listed above, nor is it affiliated with any model provider. All data is stored locally on your device; K.G.Studio does not collect or transmit your data. You are solely responsible for any data you provide to third-party model providers.
|
||||
|
||||
We hope you enjoy using K.G.Studio Musician Assistant!
|
||||
@@ -6,5 +6,6 @@ Tips:
|
||||
- Use `/clear` anytime to reset the chat.
|
||||
- Ask me to create tracks, regions, or MIDI notes, and I'll help orchestrate tool actions.
|
||||
- Type `/welcome` to view this message again.
|
||||
- Type `/help` to view the help message.
|
||||
|
||||
Have fun composing with K.G.Studio Musician Assistant.
|
||||
+18
-46
@@ -1,52 +1,24 @@
|
||||
## Welcome
|
||||
Welcome to **K.G.Studio Musician Assistant** — your in‑browser AI partner for composing and arranging.
|
||||
|
||||
Welcome to **K.G.Studio Musician Assistant**—your AI-powered agent for music composition and arrangement. Harnessing advanced language models, I am here to help you create, arrange, and refine your musical ideas with ease and intelligence.
|
||||
### Quick Start
|
||||
**Set up K.G.Studio Musician Assistant**
|
||||
- [Click here to get a free OpenRouter API Key](https://openrouter.ai/keys) (you may need an OpenRouter account).
|
||||
- In **Settings ⚙️ → General → LLM Provider**, select **OpenAI Compatible**.
|
||||
- In **OpenAI Compatible Server → Key**, paste your key. Note: on non‑localhost, your key isn’t persisted; you’ll re‑enter it after refresh. Keep it safe.
|
||||
- In **OpenAI Compatible Server → Model**, enter `qwen/qwen3-30b-a3b:free`. Note: non‑free models may require billing.
|
||||
- In **OpenAI Compatible Server → Base URL**, enter `https://openrouter.ai/api/v1/chat/completions`.
|
||||
|
||||
To get started, you will need to configure an LLM (Large Language Model) Provider in the application settings. Please follow the instructions below to ensure proper setup:
|
||||
*(Alternatively, you can use the official OpenAI API, or any OpenAI‑compatible service)*
|
||||
|
||||
---
|
||||
|
||||
### Configuring Your LLM Provider
|
||||
|
||||
Navigate to **Settings ⚙️ → General → LLM Provider**. Depending on your chosen provider, you will need to supply the appropriate API Key and, if applicable, a custom base URL (for non-official OpenAI-compatible services such as Ollama, OpenRouter, etc.).
|
||||
|
||||
---
|
||||
|
||||
### Using the OpenAI GPT Model Series
|
||||
|
||||
1. Obtain an OpenAI API Key from [**OpenAI**](https://platform.openai.com/account/api-keys). You may need to create an account and add a payment method to generate an API Key.
|
||||
2. In **Settings ⚙️ → General → LLM Provider**, select **OpenAI** as your provider.
|
||||
3. Enter your API Key in **OpenAI → Key**.
|
||||
4. Select your preferred model from the **OpenAI → Model** dropdown. For optimal performance, we recommend `gpt-5` or `gpt-4o`.
|
||||
5. Optionally, choose whether to enable Flex Mode in **OpenAI → Flex Mode**. Flex Mode offers discounted pricing, but may result in slower response times.
|
||||
|
||||
---
|
||||
|
||||
### Using OpenRouter
|
||||
|
||||
OpenRouter is a platform that provides unified access to a wide range of language models—including free options—from various providers. This makes it easy to experiment and find the model that best suits your needs.
|
||||
|
||||
1. Obtain an API Key from [**OpenRouter**](https://openrouter.ai/keys). Registration is required; for paid models, a payment method may also be necessary.
|
||||
2. In **Settings ⚙️ → General → LLM Provider**, select **OpenAI Compatible** as your provider.
|
||||
3. Enter your API Key in **OpenAI Compatible Server → Key**.
|
||||
4. Browse available models on the [**OpenRouter Models Page**](https://openrouter.ai/models). Use the "Prompt Pricing" filter to identify free models.
|
||||
**Note:** Each model provider may have different data retention and privacy policies. Please review these policies before use.
|
||||
5. Enter your chosen model name in **OpenAI Compatible Server → Model**. Recommended model series include:
|
||||
- `Google: gemini-2.5-pro`
|
||||
- `Anthropic: claude-4-sonnet`
|
||||
- `DeepSeek: deepseek-r1` (free: `deepseek/deepseek-r1-0528:free`)
|
||||
- `DeepSeek: deepseek-v3` (free: `deepseek/deepseek-chat-v3-0324:free`)
|
||||
- `Qwen: qwen3-235b-a22b` (free: `qwen/qwen3-235b-a22b:free`)
|
||||
6. Input the base URL `https://openrouter.ai/api/v1/chat/completions` of the OpenAI Compatible Server in **OpenAI Compatible Server → Base URL**.
|
||||
|
||||
---
|
||||
|
||||
### Note
|
||||
|
||||
Due to security reasons, if you are using K.G.Studio from a non-local hosted environment, we won't persist your API Key in your browser's IndexedDB in order to prevent potential XSS attack, you will need to input your API Key every time you start K.G.Studio.
|
||||
**Basic DAW operations**
|
||||
- Double‑click (or hold Ctrl/Cmd and click) on a track to create a region.
|
||||
- Drag region edges to resize; drag the body to move.
|
||||
- Click the small pencil at a region’s top‑left to open the piano roll.
|
||||
- In the piano roll, double‑click (or Ctrl/Cmd+click) to create a note.
|
||||
- Click to select; Shift+click for multi‑select; drag to box‑select.
|
||||
- Drag note edges to resize; drag the note body to move selected notes.
|
||||
- Use Snapping in the piano roll toolbar (top‑right) to quantize to grid.
|
||||
|
||||
### Disclaimer
|
||||
|
||||
K.G.Studio does not provide or host any of the models listed above, nor is it affiliated with any model provider. All data is stored locally on your device; K.G.Studio does not collect or transmit your data. You are solely responsible for any data you provide to third-party model providers.
|
||||
|
||||
We hope you enjoy using K.G.Studio Musician Assistant!
|
||||
K.G.Studio is not affiliated with any LLM provider. All data is stored locally on your device; you are responsible for any data you send to third‑party services.
|
||||
@@ -82,9 +82,36 @@ export async function processUserMessage(originalMessage: string): Promise<UserM
|
||||
}
|
||||
}
|
||||
|
||||
case '/help': {
|
||||
try {
|
||||
const url = `${import.meta.env.BASE_URL}chat/help.md`;
|
||||
const resp = await fetch(url);
|
||||
if (!resp.ok) {
|
||||
throw new Error(`Failed to fetch ${url}: ${resp.status}`);
|
||||
}
|
||||
const md = await resp.text();
|
||||
return {
|
||||
displayUserMessage: false,
|
||||
sendToLLM: false,
|
||||
finalMessageForLLM: null,
|
||||
pseudoAssistantResponse: md,
|
||||
metadata: { command: 'help' }
|
||||
};
|
||||
} catch (err) {
|
||||
const fallback = 'Help is currently unavailable.';
|
||||
return {
|
||||
displayUserMessage: false,
|
||||
sendToLLM: false,
|
||||
finalMessageForLLM: null,
|
||||
pseudoAssistantResponse: fallback,
|
||||
metadata: { command: 'help', error: String(err) }
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
default: {
|
||||
const { setStatus } = useProjectStore.getState();
|
||||
const help = 'Available commands: /clear, /welcome';
|
||||
const help = 'Available commands: /clear, /welcome, /help';
|
||||
setStatus(`Unknown command: ${command}. ${help}`);
|
||||
return {
|
||||
displayUserMessage: false,
|
||||
|
||||
+1
-1
@@ -62,7 +62,7 @@ export const wrapXmlBlocksInContent = (content: string): string => {
|
||||
if (!xmlBlocks || xmlBlocks.length === 0) return content;
|
||||
let result = content;
|
||||
for (const block of xmlBlocks) {
|
||||
const fenced = '```\n' + block + '\n```';
|
||||
const fenced = '```xml\n' + block + '\n```';
|
||||
result = result.split(block).join(fenced);
|
||||
}
|
||||
return result;
|
||||
|
||||
Reference in New Issue
Block a user