3dtours/KGStudio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add track-aware tool targeting and new agent tools, lifted the requirement of selection a MIDI region to use AI Agent

Browse Source 
Introduces a shared `toolTargeting.ts` module that resolves the active MIDI
region/track from the user's current selection, enabling tools to operate on
the correct target without requiring explicit track IDs in most cases.

Adds two new read-only tools — `list_all_tracks` and
`get_user_selected_music_range_and_track` — so the agent can inspect
available tracks and the current selection context before acting.

Also refactors `AddNotesTool` to auto-create MIDI regions when no region
exists, refactors `RemoveNotesTool`, `ReadMusicTool`, and
`ReadChordProgressionTool` to use the new targeting helpers, adds a
`buildToolHistoryContent` hook to `BaseTool` for cleaner chat history
display, and updates system prompts and tests throughout.
This commit is contained in:
Xiaohan-Tian
2026-06-04 19:29:37 -07:00
parent 5c5a07b839
commit 8ee7ddac77
27 changed files with 1812 additions and 855 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/util/abcNotationUtil.ts
+26 -5
View File
@@ -8,6 +8,7 @@ import { KGMidiNote } from '../core/midi/KGMidiNote';
import { KGCore } from '../core/KGCore';
import { KGProject } from '../core/KGProject';
import { KGChordRegion } from '../core/region/KGChordRegion';
import { FLUIDR3_INSTRUMENT_MAP } from '../constants/generalMidiConstants';
import { pitchToNoteName } from './midiUtil';
import { beatsToTicks, getTicksPerBar, reduceFraction } from './mathUtil';
import type { TimeSignature } from '../types/projectTypes';
@@ -168,18 +169,38 @@ function convertTicksToABCLength(ticks: number, timeSignature: TimeSignature): s
* @param project - Project containing tempo and time signature info
* @returns ABC header string
*/
function resolveRegionTrackMetadata(region: KGMidiRegion, project: KGProject): {
trackId: string;
trackName: string;
instrumentName: string;
} {
const trackId = region.getTrackId();
const track = project.getTracks().find(candidate => candidate.getId().toString() === trackId);
const trackName = track?.getName() || 'Unnamed Track';
const instrumentKey = 'getInstrument' in (track ?? {}) && typeof track.getInstrument === 'function'
? track.getInstrument()
: null;
const instrumentName = instrumentKey
? FLUIDR3_INSTRUMENT_MAP[instrumentKey]?.displayName || instrumentKey
: 'Unknown Instrument';
return { trackId, trackName, instrumentName };
}
function formatABCHeader(region: KGMidiRegion, project: KGProject): string {
const timeSignature = project.getTimeSignature();
const bpm = project.getBpm();
const keySignature = project.getKeySignature();
const regionName = region.getName();
const { trackId, trackName, instrumentName } = resolveRegionTrackMetadata(region, project);
// Get ABC notation key signature from the key signature map
const abcKeySignature = KEY_SIGNATURE_MAP[keySignature]?.abcNotationKeySignature || 'C';
const header = [
`track_id: ${trackId}`,
`track_name: ${trackName}`,
`Instrument: ${instrumentName}`,
'X:1', // Reference number
`T:${regionName}`, // Title
`M:${timeSignature.numerator}/${timeSignature.denominator}`, // Time signature
`L:1/${timeSignature.denominator}`, // note length unit should be aligned with time signature
`Q:1/${timeSignature.denominator}=${bpm}`, // Tempo (quarter note = BPM)
@@ -353,12 +374,12 @@ export function convertBeatRangeChordProgressionToABCNotation(
if (segments.length === 0) {
return [
'Selected Region Chord Progression',
'Chord Progression',
'This progression comes only from user-defined chord regions on the global chord track. If no chord progression is defined for this range, read the notes directly with `read_music`.',
'',
header,
'',
'No chord progression is defined for the selected MIDI region range. Use `read_music` to inspect the notes directly.'
'No chord progression is defined for the requested range on the global chord track. Use `read_music` to inspect the notes directly.'
].join('\n');
}
@@ -367,7 +388,7 @@ export function convertBeatRangeChordProgressionToABCNotation(
const chordNotes = formatChordProgressionNoteLine(segments, timeSignature);
return [
'Selected Region Chord Progression',
'Chord Progression',
'This progression comes only from user-defined chord regions on the global chord track. Representation 1 uses symbolic chord names such as `Em7b5`. Representation 2 rewrites the same progression as note-based ABC chord tokens.',
'',
header,
src/util/messageFilter/UserMessageFilter.test.ts
+12 -8
View File
@@ -35,12 +35,6 @@ vi.mock('../../stores/projectStore', () => ({
},
}));
vi.mock('../../agent/core/SystemPrompts', () => ({
SystemPrompts: {
getPromptWithContext: vi.fn(async (value: string) => value),
},
}));
vi.mock('../localLLMConfig', async () => {
const actual = await vi.importActual<typeof import('../localLLMConfig')>('../localLLMConfig');
return {
@@ -336,6 +330,17 @@ describe('processUserMessage slash commands', () => {
expect(result.metadata).toMatchObject({ error: 'local_browser_unsupported' });
});
it('passes non-command messages through to the LLM even when no region is selected', async () => {
const result = await processUserMessage('hello');
expect(result.sendToLLM).toBe(true);
expect(result.finalMessageForLLM).toBe('hello');
expect(result.pseudoAssistantResponse).toBeNull();
expect(result.metadata).toMatchObject({
mode: 'pass_through_plain_user_message',
});
});
it('allows local-browser messages when only SharedArrayBuffer isolation support is missing', async () => {
detectLocalLLMRuntimeSupportMock.mockReturnValue({
supported: true,
@@ -345,12 +350,11 @@ describe('processUserMessage slash commands', () => {
secureContext: true,
reason: 'This host may not support the local browser runtime reliably because cross-origin isolation or SharedArrayBuffer is unavailable. COOP/COEP headers may be missing.',
});
storeState.activeRegionId = 'region-1';
const result = await processUserMessage('hello');
expect(result.sendToLLM).toBe(true);
expect(result.finalMessageForLLM).toContain('hello');
expect(result.finalMessageForLLM).toBe('hello');
expect(result.pseudoAssistantResponse).toBeNull();
});
});
src/util/messageFilter/UserMessageFilter.ts
+3 -45
View File
@@ -1,7 +1,6 @@
import { clearChatHistoryAndUI } from '../chatUtil';
import { useProjectStore } from '../../stores/projectStore';
import { ConfigManager } from '../../core/config/ConfigManager';
import { SystemPrompts } from '../../agent/core/SystemPrompts';
import { detectLocalLLMRuntimeSupport, LOCAL_LLM_PROVIDER_KEY } from '../localLLMConfig';
import { normalizeLanguageSetting, resolveLanguageSetting } from '../../i18n/locale';
import type { ResolvedLocaleCode } from '../../i18n/types';
@@ -246,7 +245,7 @@ export async function processUserMessage(originalMessage: string): Promise<UserM
}
}
// Non-command message: require an active or selected region
// Non-command message: pass through to the LLM unchanged.
try {
// Provider-specific configuration checks before sending to LLM
try {
@@ -329,53 +328,12 @@ export async function processUserMessage(originalMessage: string): Promise<UserM
console.warn('Provider config check failed; proceeding with defaults', e);
}
const { activeRegionId, selectedRegionIds } = useProjectStore.getState();
const hasContextRegion = !!activeRegionId || (Array.isArray(selectedRegionIds) && selectedRegionIds.length > 0);
if (!hasContextRegion) {
// No region context: show guidance and do not send to LLM
const url = `${import.meta.env.BASE_URL}chat/error_no_selected_region.md`;
let md = 'Please select a region or open a MIDI region in the piano roll before asking for editing.';
try {
const resp = await fetch(url);
if (resp.ok) {
md = await resp.text();
}
} catch (e) {
console.warn('Failed to fetch error_no_selected_region.md', e);
// ignore fetch failure, use fallback text
}
return {
displayUserMessage: true,
sendToLLM: false,
finalMessageForLLM: null,
pseudoAssistantResponse: md,
metadata: { error: 'no_selected_region' }
};
}
// Has region context: pass through, but append processed appendix to the LLM-bound message
let appendix = '';
try {
const resp = await fetch(`${import.meta.env.BASE_URL}prompts/user_msg_appendix.md`);
if (resp.ok) {
const rawAppendix = await resp.text();
appendix = await SystemPrompts.getPromptWithContext(rawAppendix);
}
} catch (e) {
// If appendix fetch fails, proceed without it
console.warn('Failed to fetch user_msg_appendix.md', e);
}
const finalForLLM = appendix ? `${trimmed}${appendix}` : trimmed;
return {
displayUserMessage: true,
sendToLLM: true,
finalMessageForLLM: finalForLLM,
finalMessageForLLM: trimmed,
pseudoAssistantResponse: null,
metadata: { mode: 'pass_through_with_region', appendixIncluded: appendix.length > 0 }
metadata: { mode: 'pass_through_plain_user_message' }
};
} catch {
// Fallback: if store access fails, pass through unchanged