fix: MIDI Rearrange — selected item triggers dedicated tool flow

- midiExtractor.js: extractSelectedMIDIContext with note name conversion - aiGateway.js: REARRANGE_TOOL_SPEC + buildRearrangeMessage - dawCommandDispatcher.js: register REARRANGE_MIDI_MELODY command - app.jsx: handleAISend detects selected MIDI item → rearrange flow - rearrangeMidiMelody handler places A/B track below source track
2026-07-28 10:54:35 +07:00
parent 421535ca0e
commit 2eb327056d
6 changed files with 285 additions and 0 deletions
@@ -13718,6 +13718,92 @@ const App = () => {
      return;
    }

+    // ── MIDI Rearrange Flow: selected MIDI item → use rearrange tool ──
+    if (selectedItemIds && selectedItemIds.size === 1) {
+      const selId = selectedItemIds.values().next().value;
+      let isMidiItem = false;
+      let sourceTrackId = null;
+      let sourceItemName = null;
+      for (const t of activeTracks || []) {
+        const found = (t.midiItems || []).find(m => m.id === selId);
+        if (found) { isMidiItem = true; sourceTrackId = t.id; sourceItemName = found.name; break; }
+      }
+      if (isMidiItem && window.SonicMidiExtractor) {
+        try {
+          setAiProcessing(true);
+          setAiActionLog(prev => [...prev, { type: 'status', text: ` 🎯 Phát hiện MIDI item được chọn — chuyển sang chế độ Rearrange...`, time: Date.now() }]);
+
+          if (aiProviders.length === 0 || !selectedProviderId) {
+            try { const d = await window.SonicAPI.getAIConfigs(); if (d && d.providers && d.providers.length > 0) { setAiProviders(d.providers); const a = d.providers.find(p => p.is_active) || d.providers[0]; if (a) setSelectedProviderId(a.id); } } catch (e) {}
+          }
+          const prv = aiProviders.find(p => p.id === selectedProviderId) || (aiProviders.length > 0 ? aiProviders[0] : null);
+          const provider = prv || aiConfig;
+          const baseUrl = provider.api_base_url || provider.baseUrl || `${API_BASE_URL}`;
+          const apiKey = provider.api_key || provider.apiKey || '';
+          const model = provider.model_name || provider.model || 'deepseek-chat';
+          setAiActionLog(prev => [...prev, { type: 'info', text: ` Rearrange Provider: ${provider.name || 'default'} | Model: ${model}`, time: Date.now() }]);
+
+          const srcContext = window.SonicMidiExtractor.extractSelectedMIDIContext(activeTracks || tracks, selId, bpm);
+          setAiActionLog(prev => [...prev, { type: 'info', text: ` 📋 Trích xuất ${srcContext.total_notes} notes từ "${srcContext.item_name}" (${srcContext.track_name})`, time: Date.now() }]);
+
+          if (window.DAWCommandDispatcher) {
+            window.DAWCommandDispatcher.rearrangeSourceTrackId = sourceTrackId;
+            window.DAWCommandDispatcher.rearrangeSourceItemName = sourceItemName;
+          }
+
+          const messages = window.AIGateway.buildRearrangeMessage(prompt, srcContext);
+          const completion = await window.AIGateway.callLLM({
+            provider: provider.name || 'default',
+            model,
+            apiKey,
+            baseUrl: baseUrl.replace(/\/chat\/completions$/, '').replace(/\/$/, ''),
+            messages,
+            tools: [window.AIGateway.REARRANGE_TOOL_SPEC],
+            toolChoice: 'auto'
+          });
+
+          const functionCalls = window.AIGateway.extractFunctionCalls(completion);
+          if (functionCalls && functionCalls.length > 0) {
+            for (const fc of functionCalls) {
+              setAiActionLog(prev => [...prev, { type: 'info', text: ` Gọi lệnh: ${fc.name}(${JSON.stringify(fc.arguments)})`, time: Date.now() }]);
+              const cmdName = fc.name.toUpperCase();
+              if (window.DAWCommandDispatcher) {
+                try {
+                  let cmdResult = window.DAWCommandDispatcher.execute(cmdName, fc.arguments);
+                  if (cmdResult && typeof cmdResult.then === 'function') cmdResult = await cmdResult;
+                  setAiActionLog(prev => [...prev, { type: 'status', text: ` ✅ ${fc.name}: thành công — ${fc.arguments.rearrange_title || ''}`, time: Date.now() }]);
+                } catch (cmdErr) {
+                  setAiActionLog(prev => [...prev, { type: 'error', text: ` ❌ ${fc.name}: ${cmdErr.message}`, time: Date.now() }]);
+                }
+              }
+            }
+          } else {
+            setAiActionLog(prev => [...prev, { type: 'error', text: ` ❌ AI không trả về lệnh rearrange hợp lệ.`, time: Date.now() }]);
+          }
+
+          const textResp = functionCalls.length === 0 && completion.choices && completion.choices[0] && completion.choices[0].message && completion.choices[0].message.content;
+          if (textResp) {
+            setAiActionLog(prev => [...prev, { type: 'status', text: ` AI: ${textResp.slice(0, 500)}`, time: Date.now() }]);
+          }
+          if (prompt) {
+            promptHistRef.current = [...promptHistRef.current.slice(-49), prompt];
+            setPromptHistory(promptHistRef.current);
+          }
+          setPromptHistIdx(-1);
+          setAiActionLog(prev => [...prev, { type: 'status', text: ` Hoàn tất Rearrange.`, time: Date.now() }]);
+          setAiPrompt('');
+          setTimeout(() => lucide.createIcons(), 200);
+          setAiProcessing(false);
+          return;
+        } catch (err) {
+          setAiActionLog(prev => [...prev, { type: 'error', text: ` Lỗi Rearrange: ${err.message}`, time: Date.now() }]);
+          showToast(`AI Rearrange Error: ${err.message}`, 'error');
+          setAiProcessing(false);
+          return;
+        }
+      }
+    }
+
    if (window.DAWCommandDispatcher) {
      window.DAWCommandDispatcher.currentSelectedTrackId = selectedTrackId;
      window.DAWCommandDispatcher.currentTracks = activeTracks;
@@ -14670,6 +14756,77 @@ const App = () => {
        showToast(`Đã nạp ${aiTracks.length} tracks MIDI thế hệ AI!`, 'success');
        return { success: true };
      },
+      rearrangeMidiMelody: (args) => {
+        const { rearrange_title, rearranged_notes, soundfont_id, soundfont_bank, soundfont_program } = args;
+        if (!rearranged_notes || rearranged_notes.length === 0) {
+          return { success: false, error: 'No rearranged notes provided' };
+        }
+        const sourceTrackId = window.DAWCommandDispatcher?.rearrangeSourceTrackId || selectedTrackId;
+        const sourceItemName = window.DAWCommandDispatcher?.rearrangeSourceItemName || 'Source';
+        const bpmVal = parseInt(bpm) || 120;
+        const secondsPerBeat = 60.0 / bpmVal;
+        const secondsPerBar = secondsPerBeat * 4;
+
+        const totalBeats = rearranged_notes.reduce((max, n) => Math.max(max, (n.start_beat || 0) + (n.duration_beats || 1)), 0);
+        const totalBars = Math.max(1, Math.ceil(totalBeats / 4));
+        const durationSec = totalBars * secondsPerBar;
+
+        updateActiveTracks(prev => {
+          let updatedTracks = [...prev];
+          // Find source track index to insert new track right after it
+          const sourceIdx = sourceTrackId ? updatedTracks.findIndex(t => t.id === sourceTrackId || t.id === 'track_' + sourceTrackId) : -1;
+          const insertIdx = sourceIdx >= 0 ? sourceIdx + 1 : updatedTracks.length;
+          const newId = (updatedTracks.length + 1).toString();
+          const colors = ['#0f766e', '#1d4ed8', '#701a75', '#a21caf', '#b45309'];
+          const newTrack = {
+            id: newId,
+            name: `[AI Rearrange] ${rearrange_title || 'Variation'}`,
+            type: 'MIDI',
+            volumeDb: 0,
+            pan: 0,
+            muted: false,
+            solo: false,
+            color: colors[insertIdx % colors.length],
+            markers: [],
+            serverFileId: null,
+            clips: [],
+            sections: [],
+            midiItems: [{
+              id: 'item_rearr_' + Date.now() + '_' + Math.random().toString(36).substr(2, 5),
+              name: `[AI] ${rearrange_title || 'Rearranged'} - var`,
+              parent_track_id: newId,
+              startTime: currentTime,
+              duration: durationSec,
+              length_bars: totalBars,
+              notes: rearranged_notes.map((n, i) => ({
+                id: `note_rearr_${Date.now()}_${i}`,
+                pitch: Math.max(0, Math.min(127, n.pitch || 60)),
+                start_beat: Math.max(0, parseFloat(n.start_beat) || 0),
+                duration_beats: Math.max(0.125, parseFloat(n.duration_beats) || 0.25),
+                velocity: Math.max(0.1, Math.min(1.0, n.velocity ?? 0.8)),
+                pan: 0.0
+              }))
+            }]
+          };
+          if (soundfont_id || (soundfont_bank !== undefined && soundfont_program !== undefined)) {
+            newTrack.soundfont_id = soundfont_id || '';
+            newTrack.soundfont_bank = soundfont_bank ?? 0;
+            newTrack.soundfont_program = soundfont_program ?? 0;
+            newTrack.synth_engine = {
+              type: 'soundfont',
+              plugin_id: soundfont_id ? 'sf_' + soundfont_id : null,
+              soundfont_bank: soundfont_bank ?? 0,
+              soundfont_program: soundfont_program ?? 0,
+              soundfont_id: soundfont_id || ''
+            };
+          }
+          updatedTracks.splice(insertIdx, 0, newTrack);
+          return updatedTracks;
+        });
+
+        showToast(`✅ AI Rearrange: "${rearrange_title}" — ${rearranged_notes.length} notes`, 'success');
+        return { success: true, trackId: newId, totalBars };
+      },
      createMidiItem: (args) => {
        const trackId = args.track_id || selectedTrackId;
        if (!trackId) return { success: false, error: 'No track_id provided' };
@@ -88,6 +88,64 @@ const AIGateway = (function() {
    }
  }];

+  const REARRANGE_TOOL_SPEC = {
+    type: 'function',
+    function: {
+      name: 'rearrange_midi_melody',
+      description: 'Accepts source MIDI notes and rearranges/re-harmonizes them into a new musical variation while preserving the core melody. Use this when the user asks to rearrange, remix, or create variations of an existing MIDI item.',
+      parameters: {
+        type: 'object',
+        properties: {
+          rearrange_title: { type: 'string', description: 'Title/description of the rearranged version (e.g. "Jazz Variation", "Dark Orchestral Remix")' },
+          soundfont_id: { type: 'string', default: 'generaluser_gs' },
+          soundfont_bank: { type: 'integer', default: 0 },
+          soundfont_program: { type: 'integer', default: 0 },
+          rearranged_notes: {
+            type: 'array',
+            description: 'Array of rearranged MIDI notes. Keep the same total_duration_beats as the original unless user explicitly requests length change.',
+            items: {
+              type: 'object',
+              properties: {
+                pitch: { type: 'integer', description: 'MIDI note pitch 0-127 (C4=60)' },
+                start_beat: { type: 'number', description: 'Note start position in beats from 0.0' },
+                duration_beats: { type: 'number', description: 'Note length in beats (quarter=1.0)' },
+                velocity: { type: 'number', description: 'Velocity 0.0-1.0' }
+              },
+              required: ['pitch', 'start_beat', 'duration_beats', 'velocity']
+            }
+          }
+        },
+        required: ['rearrange_title', 'rearranged_notes']
+      }
+    }
+  };
+
+  function buildRearrangeMessage(prompt, sourceContext) {
+    const notesJson = JSON.stringify(sourceContext.notes, null, 2);
+    return [
+      { role: 'system', content: `You are a professional music arranger. Your task is to rearrange/re-harmonize the provided MIDI notes based on the user's request.
+
+SOURCE MIDI CONTEXT:
+- Track: ${sourceContext.track_name}
+- Item: ${sourceContext.item_name}
+- Duration: ${sourceContext.duration_bars} bars (${sourceContext.total_beats} beats)
+- BPM: ${sourceContext.bpm}
+- Total notes: ${sourceContext.total_notes}
+
+SOURCE NOTES (pitch, note_name, start_beat, duration_beats, velocity):
+${notesJson}
+
+RULES:
+1. Use the rearrange_midi_melody tool to return the rearranged notes.
+2. PRESERVE the core melodic outline and overall structure unless the user explicitly asks for a complete transformation.
+3. Keep the total duration (${sourceContext.total_beats} beats) the same unless user requests a different length.
+4. The rearranged_notes array MUST contain notes with pitch (0-127), start_beat (0.0 to ${sourceContext.total_beats}.0), duration_beats, and velocity (0.0-1.0).
+5. Start beats should remain within 0-${sourceContext.total_beats} range.
+6. You may add, remove, or modify notes to achieve the requested style.` },
+      { role: 'user', content: `User request: ${prompt}\n\nRearrange the source MIDI notes above according to this request. Return the result via the rearrange_midi_melody tool.` }
+    ];
+  }
+
  function parseOrigin(urlStr) {
    try { const u = new URL(urlStr); return `${u.protocol}//${u.hostname}${u.port ? ':'+u.port : ''}`; } catch (_) { return null; }
  }
@@ -303,6 +361,8 @@ Ví dụ: "Hãy chọn và copy từ bar 4 đến bar 12 của track 1 sau đó

  return {
    DEFAULT_TOOLS,
+    REARRANGE_TOOL_SPEC,
+    buildRearrangeMessage,
    callLLM,
    extractFunctionCalls,
    buildUserMessage,
@@ -72,6 +72,7 @@ const DAWCommandDispatcher = (function() {
    register('MODIFY_MIDI_NOTES', (args) => api.modifyMidiNotes(args));
    register('PROCESS_AI_DSP', (args) => api.processAudioDsp(args));
    register('GENERATE_MULTITRACK_MIDI', (args) => api.generateMultitrackMidi(args));
+    register('REARRANGE_MIDI_MELODY', (args) => api.rearrangeMidiMelody(args));
  }

  return {
@@ -0,0 +1,60 @@
+const SonicMidiExtractor = (function() {
+  const NOTE_NAMES = ['C','C#','D','D#','E','F','F#','G','G#','A','A#','B'];
+
+  function midiPitchToNoteName(pitch) {
+    const note = NOTE_NAMES[pitch % 12];
+    const octave = Math.floor(pitch / 12) - 1;
+    return note + octave;
+  }
+
+  function extractSelectedMIDIContext(tracks, selectedItemId, bpm) {
+    let targetItem = null;
+    let targetTrack = null;
+
+    for (const track of tracks) {
+      const items = track.midiItems || [];
+      const item = items.find(i => i.id === selectedItemId);
+      if (item) { targetItem = item; targetTrack = track; break; }
+    }
+
+    if (!targetItem) {
+      throw new Error('Please select a MIDI Item on the Timeline before requesting a Rearrangement!');
+    }
+
+    const notes = targetItem.notes || [];
+    if (notes.length === 0) {
+      throw new Error('Selected MIDI item has no notes to rearrange.');
+    }
+
+    const compactNotes = notes.map(n => ({
+      pitch: n.pitch,
+      note_name: midiPitchToNoteName(n.pitch),
+      start_beat: parseFloat((n.start_beat || 0).toFixed(2)),
+      duration_beats: parseFloat((n.duration_beats || 1).toFixed(2)),
+      velocity: parseFloat((n.velocity || 0.8).toFixed(2))
+    }));
+
+    const totalDurationBeats = compactNotes.reduce((max, n) => Math.max(max, n.start_beat + n.duration_beats), 0);
+
+    return {
+      track_name: targetTrack.name,
+      track_id: targetTrack.id,
+      item_id: targetItem.id,
+      item_name: targetItem.name,
+      duration_bars: targetItem.length_bars || Math.ceil(totalDurationBeats / 4),
+      total_beats: Math.ceil(totalDurationBeats),
+      bpm: parseInt(bpm || '120'),
+      total_notes: compactNotes.length,
+      notes: compactNotes,
+      instrument: targetTrack.synth_engine ? {
+        soundfont_id: targetTrack.synth_engine.soundfont_id || '',
+        soundfont_bank: targetTrack.synth_engine.soundfont_bank ?? 0,
+        soundfont_program: targetTrack.synth_engine.soundfont_program ?? 0
+      } : null
+    };
+  }
+
+  return { extractSelectedMIDIContext, midiPitchToNoteName };
+})();
+
+window.SonicMidiExtractor = SonicMidiExtractor;
@@ -20,6 +20,7 @@
    <script src="/static/js/services/dawCommandDispatcher.js?v=202607271016"></script>
    <script src="/static/js/services/pianoRollTabService.js?v=202607272044"></script>
    <script src="/static/js/services/ghostNoteExtractor.js?v=202607271727"></script>
+    <script src="/static/js/services/midiExtractor.js?v=202607281052"></script>
    <script src="/static/js/services/promptTemplateManager.js?v=202607281039"></script>
    <script src="/static/js/app.precompiled.js?v=202607271245" defer></script>
    <link rel="stylesheet" href="/static/css/styles.css?v=202607271016">