fix: MIDI Rearrange — selected item triggers dedicated tool flow

- midiExtractor.js: extractSelectedMIDIContext with note name conversion
- aiGateway.js: REARRANGE_TOOL_SPEC + buildRearrangeMessage
- dawCommandDispatcher.js: register REARRANGE_MIDI_MELODY command
- app.jsx: handleAISend detects selected MIDI item → rearrange flow
- rearrangeMidiMelody handler places A/B track below source track
This commit is contained in:
2026-07-28 10:54:35 +07:00
parent 421535ca0e
commit 2eb327056d
6 changed files with 285 additions and 0 deletions
+60
View File
@@ -88,6 +88,64 @@ const AIGateway = (function() {
}
}];
const REARRANGE_TOOL_SPEC = {
type: 'function',
function: {
name: 'rearrange_midi_melody',
description: 'Accepts source MIDI notes and rearranges/re-harmonizes them into a new musical variation while preserving the core melody. Use this when the user asks to rearrange, remix, or create variations of an existing MIDI item.',
parameters: {
type: 'object',
properties: {
rearrange_title: { type: 'string', description: 'Title/description of the rearranged version (e.g. "Jazz Variation", "Dark Orchestral Remix")' },
soundfont_id: { type: 'string', default: 'generaluser_gs' },
soundfont_bank: { type: 'integer', default: 0 },
soundfont_program: { type: 'integer', default: 0 },
rearranged_notes: {
type: 'array',
description: 'Array of rearranged MIDI notes. Keep the same total_duration_beats as the original unless user explicitly requests length change.',
items: {
type: 'object',
properties: {
pitch: { type: 'integer', description: 'MIDI note pitch 0-127 (C4=60)' },
start_beat: { type: 'number', description: 'Note start position in beats from 0.0' },
duration_beats: { type: 'number', description: 'Note length in beats (quarter=1.0)' },
velocity: { type: 'number', description: 'Velocity 0.0-1.0' }
},
required: ['pitch', 'start_beat', 'duration_beats', 'velocity']
}
}
},
required: ['rearrange_title', 'rearranged_notes']
}
}
};
function buildRearrangeMessage(prompt, sourceContext) {
const notesJson = JSON.stringify(sourceContext.notes, null, 2);
return [
{ role: 'system', content: `You are a professional music arranger. Your task is to rearrange/re-harmonize the provided MIDI notes based on the user's request.
SOURCE MIDI CONTEXT:
- Track: ${sourceContext.track_name}
- Item: ${sourceContext.item_name}
- Duration: ${sourceContext.duration_bars} bars (${sourceContext.total_beats} beats)
- BPM: ${sourceContext.bpm}
- Total notes: ${sourceContext.total_notes}
SOURCE NOTES (pitch, note_name, start_beat, duration_beats, velocity):
${notesJson}
RULES:
1. Use the rearrange_midi_melody tool to return the rearranged notes.
2. PRESERVE the core melodic outline and overall structure unless the user explicitly asks for a complete transformation.
3. Keep the total duration (${sourceContext.total_beats} beats) the same unless user requests a different length.
4. The rearranged_notes array MUST contain notes with pitch (0-127), start_beat (0.0 to ${sourceContext.total_beats}.0), duration_beats, and velocity (0.0-1.0).
5. Start beats should remain within 0-${sourceContext.total_beats} range.
6. You may add, remove, or modify notes to achieve the requested style.` },
{ role: 'user', content: `User request: ${prompt}\n\nRearrange the source MIDI notes above according to this request. Return the result via the rearrange_midi_melody tool.` }
];
}
function parseOrigin(urlStr) {
try { const u = new URL(urlStr); return `${u.protocol}//${u.hostname}${u.port ? ':'+u.port : ''}`; } catch (_) { return null; }
}
@@ -303,6 +361,8 @@ Ví dụ: "Hãy chọn và copy từ bar 4 đến bar 12 của track 1 sau đó
return {
DEFAULT_TOOLS,
REARRANGE_TOOL_SPEC,
buildRearrangeMessage,
callLLM,
extractFunctionCalls,
buildUserMessage,
@@ -72,6 +72,7 @@ const DAWCommandDispatcher = (function() {
register('MODIFY_MIDI_NOTES', (args) => api.modifyMidiNotes(args));
register('PROCESS_AI_DSP', (args) => api.processAudioDsp(args));
register('GENERATE_MULTITRACK_MIDI', (args) => api.generateMultitrackMidi(args));
register('REARRANGE_MIDI_MELODY', (args) => api.rearrangeMidiMelody(args));
}
return {
+60
View File
@@ -0,0 +1,60 @@
const SonicMidiExtractor = (function() {
const NOTE_NAMES = ['C','C#','D','D#','E','F','F#','G','G#','A','A#','B'];
function midiPitchToNoteName(pitch) {
const note = NOTE_NAMES[pitch % 12];
const octave = Math.floor(pitch / 12) - 1;
return note + octave;
}
function extractSelectedMIDIContext(tracks, selectedItemId, bpm) {
let targetItem = null;
let targetTrack = null;
for (const track of tracks) {
const items = track.midiItems || [];
const item = items.find(i => i.id === selectedItemId);
if (item) { targetItem = item; targetTrack = track; break; }
}
if (!targetItem) {
throw new Error('Please select a MIDI Item on the Timeline before requesting a Rearrangement!');
}
const notes = targetItem.notes || [];
if (notes.length === 0) {
throw new Error('Selected MIDI item has no notes to rearrange.');
}
const compactNotes = notes.map(n => ({
pitch: n.pitch,
note_name: midiPitchToNoteName(n.pitch),
start_beat: parseFloat((n.start_beat || 0).toFixed(2)),
duration_beats: parseFloat((n.duration_beats || 1).toFixed(2)),
velocity: parseFloat((n.velocity || 0.8).toFixed(2))
}));
const totalDurationBeats = compactNotes.reduce((max, n) => Math.max(max, n.start_beat + n.duration_beats), 0);
return {
track_name: targetTrack.name,
track_id: targetTrack.id,
item_id: targetItem.id,
item_name: targetItem.name,
duration_bars: targetItem.length_bars || Math.ceil(totalDurationBeats / 4),
total_beats: Math.ceil(totalDurationBeats),
bpm: parseInt(bpm || '120'),
total_notes: compactNotes.length,
notes: compactNotes,
instrument: targetTrack.synth_engine ? {
soundfont_id: targetTrack.synth_engine.soundfont_id || '',
soundfont_bank: targetTrack.synth_engine.soundfont_bank ?? 0,
soundfont_program: targetTrack.synth_engine.soundfont_program ?? 0
} : null
};
}
return { extractSelectedMIDIContext, midiPitchToNoteName };
})();
window.SonicMidiExtractor = SonicMidiExtractor;