feat: add track-aware tool targeting and new agent tools, lifted the requirement of selection a MIDI region to use AI Agent
Introduces a shared `toolTargeting.ts` module that resolves the active MIDI region/track from the user's current selection, enabling tools to operate on the correct target without requiring explicit track IDs in most cases. Adds two new read-only tools — `list_all_tracks` and `get_user_selected_music_range_and_track` — so the agent can inspect available tracks and the current selection context before acting. Also refactors `AddNotesTool` to auto-create MIDI regions when no region exists, refactors `RemoveNotesTool`, `ReadMusicTool`, and `ReadChordProgressionTool` to use the new targeting helpers, adds a `buildToolHistoryContent` hook to `BaseTool` for cleaner chat history display, and updates system prompts and tests throughout.
This commit is contained in:
@@ -54,6 +54,7 @@ vi.mock('../utils/chatMessageUtils', () => ({
|
||||
import { AgentCore } from '../agent/core/AgentCore';
|
||||
import { KGCore } from '../core/KGCore';
|
||||
import { KGMidiRegion } from '../core/region/KGMidiRegion';
|
||||
import { KGMidiTrack } from '../core/track/KGMidiTrack';
|
||||
import { useStreamProcessor } from './useStreamProcessor';
|
||||
import type { ChatMessage } from '../types/projectTypes';
|
||||
|
||||
@@ -64,6 +65,7 @@ const flushMicrotasks = async (): Promise<void> => {
|
||||
|
||||
describe('useStreamProcessor', () => {
|
||||
beforeEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
mockedStoreState.activeRegionId = null;
|
||||
mockedStoreState.toolFastForwardEnabled = false;
|
||||
mockedStoreState.setToolFastForwardEnabled.mockClear();
|
||||
@@ -297,16 +299,12 @@ describe('useStreamProcessor', () => {
|
||||
},
|
||||
} as unknown as AgentCore);
|
||||
|
||||
const leadTrack = new KGMidiTrack('Lead', 1);
|
||||
leadTrack.setRegions([selectedRegion]);
|
||||
vi.spyOn(KGCore, 'instance').mockReturnValue({
|
||||
getCurrentProject: () => ({
|
||||
getTimeSignature: () => ({ numerator: 4, denominator: 4 }),
|
||||
getTracks: () => [
|
||||
{
|
||||
getId: () => '1',
|
||||
getName: () => 'Lead',
|
||||
getRegions: () => [selectedRegion],
|
||||
},
|
||||
],
|
||||
getTracks: () => [leadTrack],
|
||||
}),
|
||||
getSelectedItems: () => [selectedRegion],
|
||||
} as unknown as KGCore);
|
||||
@@ -406,6 +404,71 @@ describe('useStreamProcessor', () => {
|
||||
expect(toolResultMessage?.toolResultDisplayContent).toBe('raw music result');
|
||||
});
|
||||
|
||||
it('uses tool-specific history and UI strings for error results when provided', async () => {
|
||||
vi.spyOn(AgentCore, 'instance').mockReturnValue({
|
||||
getAgentState: () => ({
|
||||
getTodos: () => [],
|
||||
}),
|
||||
processUserInput: async function* () {
|
||||
yield {
|
||||
type: 'tool_call',
|
||||
content: '',
|
||||
toolCall: {
|
||||
id: 'add-call-no-target',
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'add_notes',
|
||||
arguments: JSON.stringify({
|
||||
notes: [{ pitch: 'C4', start: 0, length: 1 }],
|
||||
}),
|
||||
},
|
||||
},
|
||||
};
|
||||
yield {
|
||||
type: 'tool_result',
|
||||
content: '',
|
||||
toolResult: {
|
||||
toolCallId: 'add-call-no-target',
|
||||
name: 'add_notes',
|
||||
success: false,
|
||||
result: 'No MIDI target could be resolved. Select the MIDI region you want me to edit and retry, or tell me which MIDI track to operate on by providing its track_id.',
|
||||
},
|
||||
};
|
||||
yield { type: 'done', content: '' };
|
||||
},
|
||||
} as unknown as AgentCore);
|
||||
|
||||
const messages = new Map<string, ChatMessage>();
|
||||
|
||||
const { result } = renderHook(() => useStreamProcessor({
|
||||
onMessageAdd: (message) => {
|
||||
messages.set(message.id, message);
|
||||
},
|
||||
onMessageUpdate: (messageId, updater) => {
|
||||
const current = messages.get(messageId);
|
||||
if (!current) {
|
||||
throw new Error(`Missing message ${messageId}`);
|
||||
}
|
||||
messages.set(messageId, updater(current));
|
||||
},
|
||||
onMessageRemove: (messageId) => {
|
||||
messages.delete(messageId);
|
||||
},
|
||||
onProcessingChange: () => undefined,
|
||||
}));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.processStream('add notes prompt');
|
||||
});
|
||||
|
||||
const addNotesMessage = [...messages.values()].find(message => message.toolName === 'add_notes');
|
||||
expect(addNotesMessage?.toolRawResult).toBe(
|
||||
'No MIDI target could be resolved. Select the MIDI region you want me to edit and retry, or tell me which MIDI track to operate on by providing its track_id.',
|
||||
);
|
||||
expect(addNotesMessage?.content).toContain('I could not tell which MIDI content to edit.');
|
||||
expect(addNotesMessage?.toolResultDisplayContent).toBe('Select a MIDI region, or specify a track.');
|
||||
});
|
||||
|
||||
it('shows a confirmation card and replaces it with a denied result when the user denies execution', async () => {
|
||||
vi.spyOn(AgentCore, 'instance').mockReturnValue({
|
||||
getAgentState: () => ({
|
||||
@@ -440,16 +503,12 @@ describe('useStreamProcessor', () => {
|
||||
} as unknown as AgentCore);
|
||||
|
||||
const selectedRegion = new KGMidiRegion('region-1', '1', 0, 'Verse Melody');
|
||||
const leadTrack = new KGMidiTrack('Lead', 1);
|
||||
leadTrack.setRegions([selectedRegion]);
|
||||
vi.spyOn(KGCore, 'instance').mockReturnValue({
|
||||
getCurrentProject: () => ({
|
||||
getTimeSignature: () => ({ numerator: 4, denominator: 4 }),
|
||||
getTracks: () => [
|
||||
{
|
||||
getId: () => '1',
|
||||
getName: () => 'Lead',
|
||||
getRegions: () => [selectedRegion],
|
||||
},
|
||||
],
|
||||
getTracks: () => [leadTrack],
|
||||
}),
|
||||
getSelectedItems: () => [selectedRegion],
|
||||
} as unknown as KGCore);
|
||||
@@ -481,7 +540,7 @@ describe('useStreamProcessor', () => {
|
||||
|
||||
const confirmationMessage = [...messages.values()].find(message => message.toolConfirmation);
|
||||
expect(confirmationMessage?.toolConfirmation?.toolName).toBe('add_notes');
|
||||
expect(confirmationMessage?.toolConfirmation?.message).toContain('Allow creating 1 note in region **Verse Melody**');
|
||||
expect(confirmationMessage?.toolConfirmation?.message).toContain('Allow creating 1 note on track **Lead** in region **Verse Melody**');
|
||||
|
||||
act(() => {
|
||||
confirmationMessage?.onToolConfirmationDecision?.('deny');
|
||||
@@ -533,16 +592,12 @@ describe('useStreamProcessor', () => {
|
||||
} as unknown as AgentCore);
|
||||
|
||||
const selectedRegion = new KGMidiRegion('region-1', '1', 0, 'Intro');
|
||||
const leadTrack = new KGMidiTrack('Lead', 1);
|
||||
leadTrack.setRegions([selectedRegion]);
|
||||
vi.spyOn(KGCore, 'instance').mockReturnValue({
|
||||
getCurrentProject: () => ({
|
||||
getTimeSignature: () => ({ numerator: 4, denominator: 4 }),
|
||||
getTracks: () => [
|
||||
{
|
||||
getId: () => '1',
|
||||
getName: () => 'Lead',
|
||||
getRegions: () => [selectedRegion],
|
||||
},
|
||||
],
|
||||
getTracks: () => [leadTrack],
|
||||
}),
|
||||
getSelectedItems: () => [selectedRegion],
|
||||
} as unknown as KGCore);
|
||||
|
||||
@@ -174,17 +174,21 @@ export const useStreamProcessor = (options: StreamProcessorOptions): StreamProce
|
||||
const pendingToolCall = pendingToolCallIndex >= 0
|
||||
? pendingToolCalls.splice(pendingToolCallIndex, 1)[0]
|
||||
: undefined;
|
||||
let toolHistoryContent = result;
|
||||
let toolResultDisplayContent = result;
|
||||
if (success) {
|
||||
try {
|
||||
const toolInstance = createToolInstance(name);
|
||||
toolResultDisplayContent = toolInstance?.buildToolResultDisplayContent(
|
||||
pendingToolCall?.arguments ?? null,
|
||||
{ success, result },
|
||||
) ?? result;
|
||||
} catch {
|
||||
toolResultDisplayContent = result;
|
||||
}
|
||||
try {
|
||||
const toolInstance = createToolInstance(name);
|
||||
toolHistoryContent = toolInstance?.buildToolHistoryContent(
|
||||
pendingToolCall?.arguments ?? null,
|
||||
{ success, result },
|
||||
) ?? result;
|
||||
toolResultDisplayContent = toolInstance?.buildToolResultDisplayContent(
|
||||
pendingToolCall?.arguments ?? null,
|
||||
{ success, result },
|
||||
) ?? result;
|
||||
} catch {
|
||||
toolHistoryContent = result;
|
||||
toolResultDisplayContent = result;
|
||||
}
|
||||
const toolResultMsg = name === TODO_TOOL_NAME
|
||||
? {
|
||||
@@ -194,7 +198,7 @@ export const useStreamProcessor = (options: StreamProcessorOptions): StreamProce
|
||||
todoSnapshot: AgentCore.instance().getAgentState().getTodos().map(todo => ({ ...todo })),
|
||||
}
|
||||
: {
|
||||
...createMessage('assistant', `${success ? '✅' : '❌'} **${name}**\n\n └── ${result}`),
|
||||
...createMessage('assistant', `${success ? '✅' : '❌'} **${name}**\n\n └── ${toolHistoryContent}`),
|
||||
toolName: name,
|
||||
toolSuccess: success,
|
||||
toolRawResult: result,
|
||||
|
||||
Reference in New Issue
Block a user