IMPROVE: cài đặt thêm tool để có thể phối lại cho đoạn midi item hoặc viết thêm cho nó

This commit is contained in:
2026-08-04 15:23:11 +07:00
parent fc663921ff
commit c047934fc4
5 changed files with 375 additions and 18 deletions
+225 -5
View File
@@ -769,14 +769,14 @@ function createEqProModule(ctx, params) {
const f = filters[i]; if (!f) return;
const now = ctx.currentTime;
if (patch.type !== undefined) f.type = patch.type;
if (patch.freq !== undefined) f.frequency.setTargetAtTime(eqproClamp(b.freq, EQPRO_F_MIN, EQPRO_F_MAX), now, 0.005);
if (patch.q !== undefined) f.Q.setTargetAtTime(eqproClamp(b.q, 0.1, 18), now, 0.005);
if (patch.gain !== undefined || patch.active !== undefined) f.gain.setTargetAtTime((b.active !== false) ? ((b.gain || 0) * amount / 100) : 0, now, 0.005);
if (patch.freq !== undefined) f.frequency.setValueAtTime(eqproClamp(b.freq, EQPRO_F_MIN, EQPRO_F_MAX), now);
if (patch.q !== undefined) f.Q.setValueAtTime(eqproClamp(b.q, 0.1, 18), now);
if (patch.gain !== undefined || patch.active !== undefined) f.gain.setValueAtTime((b.active !== false) ? ((b.gain || 0) * amount / 100) : 0, now);
};
const setAmount = (a) => {
amount = eqproClamp(a, 0, 200);
const now = ctx.currentTime;
filters.forEach((f, i) => { const b = bands[i]; if (b) f.gain.setTargetAtTime((b.active !== false) ? ((b.gain || 0) * amount / 100) : 0, now, 0.005); });
filters.forEach((f, i) => { const b = bands[i]; if (b) f.gain.setValueAtTime((b.active !== false) ? ((b.gain || 0) * amount / 100) : 0, now); });
};
// Replace the internal band model + rebuild DSP immediately (add/delete/reset
// from the UI) guarantees the audible result matches the added bands at once.
@@ -13955,6 +13955,7 @@ const App = () => {
isRunning: false,
});
const [aiPrompt, setAiPrompt] = useState('');
const [aiComposeMode, setAiComposeMode] = useState('SIMILAR_VARIATION');
const [promptHistory, setPromptHistory] = useState([]);
const [promptHistIdx, setPromptHistIdx] = useState(-1);
const promptHistRef = useRef([]);
@@ -14083,7 +14084,7 @@ const App = () => {
const tlist = activeTracks || tracks || [];
for (const t of tlist) {
const found = (t.midiItems || []).find(m => m.id === selId);
if (found) return { itemName: found.name, trackName: t.name, trackId: t.id, itemId: selId };
if (found) return { itemName: found.name, trackName: t.name, trackId: t.id, itemId: selId, notes: found.notes || [], startTime: found.startTime || 0, duration: found.duration || 4 };
}
return null;
};
@@ -22507,6 +22508,206 @@ const App = () => {
}
};
// AI Compose Similar NEXT TRACK (user request)
// Click MIDI item + gõ prompt AI phân tích cu trúc giai điu trong item
// (pitch range, intervals, rhythm) sáng tác giai điu TƯƠNG T ghi MIDI
// notes vào track K TIP; nếu track kế tiếp KHÔNG RNG chèn track mi
// (ngay sau track hin ti) và ghi MIDI item vào đó.
// AI Compose (ai_midi_rearrange_specification.md)
// Click MIDI item + chn mode (Variation/Extend) + gõ prompt:
// 1. SonicMidiExtractor nén note (pitch + note_name, round 2dp) tiết kim token
// 2. Function Tool Schema rearrange_or_extend_midi_melody (2 mode)
// 3. Prompt engineering: ORIGINAL CONTEXT + USER DIRECTIVE + MODE DIRECTIVE + CONSTRAINTS
// 4. Ingest: SIMILAR_VARIATION track mi ngay dưi (A/B); EXTEND_CONTINUATION append item sau item gc
const handleAiComposeFromItem = async (mode) => {
const selMidi = getSelectedMidiItemInfo();
if (!selMidi) { showToast('Hãy chọn 1 MIDI item trên timeline trước.', 'warning'); return; }
const prompt = aiPrompt.trim();
if (!prompt) { showToast('Vui lòng gõ prompt yêu cầu giai điệu.', 'warning'); return; }
const composeMode = mode === 'EXTEND_CONTINUATION' ? 'EXTEND_CONTINUATION' : 'SIMILAR_VARIATION';
setAiProcessing(true);
setAiActionLog(prev => [...prev, { type: 'status', text: ` 🎵 Đang phân tích "${selMidi.itemName}" (${(selMidi.notes || []).length} notes) + gửi AI [${composeMode === 'EXTEND_CONTINUATION' ? 'Extend' : 'Variation'}]...`, time: Date.now() }]);
try {
if (aiProviders.length === 0 || !selectedProviderId) {
try { const d = await window.SonicAPI.getAIConfigs(); if (d && d.providers && d.providers.length > 0) { setAiProviders(d.providers); const a = d.providers.find(p => p.is_active) || d.providers[0]; if (a) setSelectedProviderId(a.id); } } catch (e) {}
}
const prv = aiProviders.find(p => p.id === selectedProviderId) || (aiProviders.length > 0 ? aiProviders[0] : null);
const provider = prv || aiConfig;
const baseUrl = provider.api_base_url || provider.baseUrl || `${API_BASE_URL}`;
const apiKey = provider.api_key || provider.apiKey || '';
const model = provider.model_name || provider.model || 'deepseek-chat';
setAiActionLog(prev => [...prev, { type: 'info', text: ` Provider: ${provider.name || 'default'} | Model: ${model}`, time: Date.now() }]);
// STEP 1: extract + nén MIDI context (note_name, round 2dp) qua SonicMidiExtractor
if (!window.SonicMidiExtractor) throw new Error('SonicMidiExtractor chưa được nạp.');
const midiContext = window.SonicMidiExtractor.extractSelectedMIDIContext(activeTracksRef.current || activeTracks, selMidi.itemId, bpm);
const bpmVal = parseInt(bpm) || 120;
const secPerBar = (60.0 / bpmVal) * 4;
const srcItem = (activeTracksRef.current || activeTracks).flatMap(t => (t.midiItems || []).map(m => ({ ...m, trackId: t.id }))).find(m => m.id === selMidi.itemId);
const startBar = srcItem ? (srcItem.startTime || 0) / secPerBar : 0;
midiContext.start_bar = startBar;
midiContext.time_signature = '4/4';
// STEP 3: prompt engineering (spec §4)
const notesJSON = JSON.stringify(midiContext.notes, null, 2);
const nextStartBar = startBar + midiContext.duration_bars;
const modeDirective = composeMode === 'EXTEND_CONTINUATION'
? `MODE: MELODIC EXTENSION (EXTEND / CONTINUATION)
- Source Section: Bars ${startBar.toFixed(2)} -> ${nextStartBar.toFixed(2)} (${midiContext.duration_bars} bars).
- DIRECTIVE: Write a CONTINUATION starting at Bar ${nextStartBar.toFixed(2)} spanning the next ${midiContext.duration_bars} bars.
- Analyze the ending motif/cadence of the original sequence to craft a smooth transition, then develop the melody toward a climax or resolution.
- Set target_start_bar = ${nextStartBar.toFixed(2)}, target_duration_bars = ${midiContext.duration_bars}.`
: `MODE: REARRANGEMENT / VARIATION (SIMILAR VARIATION / REARRANGE)
- DIRECTIVE: Generate a NEW variation of EQUAL LENGTH (${midiContext.duration_bars} bars, spanning Beats 0.0 -> ${midiContext.total_beats}).
- Retain the core harmonic framework/chord progression while applying variation techniques (Syncopation, Arpeggiator, Passing tones, Swing feel, Harmonization) as requested by the user.
- Set target_start_bar = ${startBar.toFixed(2)}, target_duration_bars = ${midiContext.duration_bars}.`;
const promptText = `You are a professional AI Music Composer & Arranger.
ORIGINAL MIDI MELODY CONTEXT FROM DAW:
- Track: "${midiContext.track_name}" | Item: "${midiContext.item_name}"
- Tempo: ${midiContext.bpm} BPM | Time Signature: ${midiContext.time_signature}
- Location: Bar ${startBar.toFixed(2)} (Length: ${midiContext.duration_bars} bars / ${midiContext.total_beats} beats)
- Array of ${midiContext.total_notes} original MIDI source notes:
${notesJSON}
USER DIRECTIVE:
"${prompt}"
${modeDirective}
STRICT CONSTRAINTS:
1. Return your answer as a SINGLE valid JSON object (NO markdown, NO code fences, NO explanation) with EXACTLY this shape:
{"mode":"SIMILAR_VARIATION or EXTEND_CONTINUATION","composition_title":"short style title","target_start_bar":number,"target_duration_bars":number,"generated_notes":[{"pitch":60,"start_beat":0.0,"duration_beats":1.0,"velocity":0.8}]}
2. Notes in generated_notes MUST cover the full beat range from 0.0 to ${midiContext.total_beats} without stopping prematurely.
3. Ensure the final note sustains or resolves cleanly at beat ${midiContext.total_beats}.
4. generated_notes start_beat is relative to the start of the generated item (beat 0.0 = first bar of the new section).`;
// STEP 2 + dispatch: provider hin ti tr tool-call rác (_unknown) nên
// KHÔNG gi tools yêu cu JSON thun trong prompt (text-based, ging
// flow Piano Roll đã hot đng).
const result = await window.AIGateway.executeAIPrompt({
prompt: promptText,
provider: provider.name || 'default',
model: model,
apiKey: apiKey,
baseUrl: baseUrl.replace(/\/chat\/completions$/, '').replace(/\/$/, ''),
systemInstruction: 'You are a professional AI Music Composer & Arranger. Output ONLY a single valid JSON object (no markdown, no code fences) matching the exact shape requested in the prompt.',
tools: []
});
if (!result) throw new Error('AI không phản hồi');
// STEP 4: decode ƯU TIÊN textResponse (JSON thun); functionCalls ch
// dùng khi đúng tên tool rearrange_or_extend_midi_melody (provider cũ
// tr tool-call rác _unknown ch cha "reason" b qua).
let aiResult = null;
let rawDebug = '';
const pickNotes = obj => {
if (!obj) return null;
if (Array.isArray(obj)) return obj;
for (const k of ['generated_notes', 'notes', 'rearranged_notes', 'midi_notes']) {
if (Array.isArray(obj[k]) && obj[k].length > 0) return obj[k];
}
return null;
};
const textResp = result.textResponse || result.text || '';
if (textResp && typeof textResp === 'string' && textResp.trim()) {
try {
const cleaned = textResp.replace(/```json?\s*/g, '').replace(/```/g, '').trim();
const parsed = JSON.parse(cleaned);
const n = pickNotes(parsed);
if (n) aiResult = { mode: parsed.mode, composition_title: parsed.composition_title, target_start_bar: parsed.target_start_bar, target_duration_bars: parsed.target_duration_bars, generated_notes: n };
} catch (e1) { rawDebug = 'text: ' + textResp.slice(0, 400); console.error('Parse AI result error:', e1); }
}
if (!aiResult || !pickNotes(aiResult)) {
if (result.functionCalls && result.functionCalls.length > 0) {
const real = result.functionCalls.find(fc => fc.name === 'rearrange_or_extend_midi_melody');
if (real) {
try {
const a = typeof real.arguments === 'string' ? JSON.parse(real.arguments) : real.arguments;
const n = pickNotes(a);
if (n) aiResult = { mode: a.mode, composition_title: a.composition_title, target_start_bar: a.target_start_bar, target_duration_bars: a.target_duration_bars, generated_notes: n };
} catch (e2) { rawDebug += ' | FC-args-raw: ' + String(real.arguments).slice(0, 300); }
} else {
rawDebug += ' | FC names: ' + result.functionCalls.map(fc => fc.name || '_unknown').join(',') + ' (không phải tool đúng — bỏ qua)';
}
}
}
if (!aiResult || !pickNotes(aiResult)) {
console.error('[AI Compose] Không parse được notes. Debug:', rawDebug, '| functionCalls:', JSON.stringify(result.functionCalls || []).slice(0, 500), '| textResponse:', (result.textResponse || result.text || '').slice(0, 500));
setAiActionLog(prev => [...prev, { type: 'error', text: ' ❌ AI không trả về notes hợp lệ. ' + (rawDebug ? 'Raw: ' + rawDebug.slice(0, 200) : ''), time: Date.now() }]);
return;
}
const aiNotes = pickNotes(aiResult);
const finalMode = aiResult.mode === 'EXTEND_CONTINUATION' ? 'EXTEND_CONTINUATION' : 'SIMILAR_VARIATION';
const title = aiResult.composition_title || 'AI Composition';
const targetStartBar = (typeof aiResult.target_start_bar === 'number' ? aiResult.target_start_bar : (finalMode === 'EXTEND_CONTINUATION' ? nextStartBar : startBar));
const targetDurBars = (typeof aiResult.target_duration_bars === 'number' ? aiResult.target_duration_bars : midiContext.duration_bars);
const newNotes = aiNotes.map((n, i) => ({
id: 'note_ai_' + Date.now() + '_' + i,
pitch: Math.max(0, Math.min(127, parseInt(n.pitch) || 60)),
start_beat: Math.max(0, parseFloat(n.start_beat) || 0),
duration_beats: Math.max(0.125, parseFloat(n.duration_beats) || 0.25),
velocity: Math.max(0.1, Math.min(1.0, n.velocity ?? 0.8)),
pan: 0.0
}));
const newItem = {
id: 'item_ai_' + Date.now(),
startTime: Math.max(0, targetStartBar * secPerBar),
duration: Math.max(1, targetDurBars * secPerBar),
name: (finalMode === 'EXTEND_CONTINUATION' ? '[Extend] ' : '[Variation] ') + title,
notes: newNotes
};
// STEP 5: ingest Variation track mi ngay dưi (A/B); Extend append item cui track gc
const curTracks = activeTracksRef.current || activeTracks;
const srcIdx = curTracks.findIndex(t => t.id === selMidi.trackId);
if (srcIdx === -1) { setAiActionLog(prev => [...prev, { type: 'error', text: ' ❌ Không tìm thấy track nguồn.', time: Date.now() }]); return; }
const sf = aiResult.soundfont_id || (midiContext.instrument && midiContext.instrument.soundfont_id) || 'generaluser_gs';
const sfBank = aiResult.soundfont_bank ?? ((midiContext.instrument && midiContext.instrument.soundfont_bank) ?? 0);
const sfProg = aiResult.soundfont_program ?? ((midiContext.instrument && midiContext.instrument.soundfont_program) ?? 0);
if (finalMode === 'EXTEND_CONTINUATION') {
// Append item mi vào CUI track gc
updateActiveTracks(prev => prev.map(t => t.id === selMidi.trackId ? { ...t, midiItems: [...(t.midiItems || []), newItem] } : t));
setAiActionLog(prev => [...prev, { type: 'status', text: ` ✅ [Extend] "${newItem.name}" (${newNotes.length} notes) → append sau item gốc (bar ${targetStartBar.toFixed(2)}).`, time: Date.now() }]);
} else {
// To track mi ngay dưi track ngun (A/B testing) spec §5.
// TUÂN TH RULES như track gc: clone toàn b cu trúc track ngun
// (synth_engine, instrumentProgram/instrumentName, volumeDb, pan,
// fxActive, bypass flags, color...) ri reset phn content.
const newTrackId = 'track_ai_var_' + Date.now();
const srcTrack = curTracks[srcIdx];
const newTrack = {
...(srcTrack || {}),
id: newTrackId,
name: '[AI Var] ' + title,
buffer: null,
startTime: 0,
clips: [],
sections: [],
midiItems: [newItem],
muted: false,
solo: false,
markers: [],
serverFileId: null,
synth_engine: srcTrack && srcTrack.synth_engine ? srcTrack.synth_engine : { type: 'soundfont', soundfont_id: sf, soundfont_bank: sfBank, soundfont_program: sfProg },
instrumentProgram: srcTrack ? srcTrack.instrumentProgram : sfProg,
instrumentName: srcTrack ? srcTrack.instrumentName : ('AI ' + title)
};
updateActiveTracks(prev => { const copy = [...prev]; copy.splice(srcIdx + 1, 0, newTrack); return copy; });
setAiActionLog(prev => [...prev, { type: 'status', text: ` ✅ [Variation] "${newItem.name}" (${newNotes.length} notes) → track mới "[AI Var] ${title}" ngay dưới (A/B) — kế thừa synth/instrument của track gốc.`, time: Date.now() }]);
}
setCanvasRedrawCount(n => n + 1);
setTimeout(() => lucide.createIcons(), 200);
} catch (err) {
setAiActionLog(prev => [...prev, { type: 'error', text: ` ❌ Lỗi: ${err.message}`, time: Date.now() }]);
showToast(`AI Error: ${err.message}`, 'error');
} finally {
setAiProcessing(false);
}
};
// Alias cũ gi cho nút UI hin ti (Compose) hot đng mode mc đnh Variation
const handleAiComposeToNextTrack = () => handleAiComposeFromItem('SIMILAR_VARIATION');
// Split Track at Playhead
const handleSplitTrackAtTime = (trackId, clipId, time) => {
const track = tracks.find(t => t.id === trackId);
@@ -24895,6 +25096,25 @@ const App = () => {
"data-lucide": "send",
className: "w-3 h-3"
})), " Gửi")), /*#__PURE__*/React.createElement("button", {
onClick: () => setAiComposeMode(aiComposeMode === 'EXTEND_CONTINUATION' ? 'SIMILAR_VARIATION' : 'EXTEND_CONTINUATION'),
title: "Chọn mode AI: Variation (biến tấu cùng độ dài) / Extend (viết tiếp các bar sau)",
className: "px-2 py-1 rounded text-[10px] border font-semibold flex items-center justify-center gap-1 " + (aiComposeMode === 'EXTEND_CONTINUATION' ? 'bg-sky-800 text-sky-100 border-sky-600' : 'bg-amber-800 text-amber-100 border-amber-600')
}, /*#__PURE__*/React.createElement("span", {
className: "inline-flex items-center shrink-0"
}, /*#__PURE__*/React.createElement("i", {
"data-lucide": aiComposeMode === 'EXTEND_CONTINUATION' ? "arrow-right" : "shuffle",
className: "w-3 h-3"
})), aiComposeMode === 'EXTEND_CONTINUATION' ? 'Extend' : 'Variation'), /*#__PURE__*/React.createElement("button", {
onClick: () => handleAiComposeFromItem(aiComposeMode),
disabled: aiProcessing,
title: "AI phân tích giai điệu MIDI item đang chọn → sáng tác theo mode đã chọn → ghi kết quả (Variation: track mới ngay dưới A/B; Extend: append sau item gốc)",
className: "px-2 py-1 bg-teal-800 hover:bg-teal-700 text-white rounded text-[10px] border border-teal-600 flex items-center justify-center gap-1" + (hasSelItem ? '' : ' opacity-40 pointer-events-none')
}, /*#__PURE__*/React.createElement("span", {
className: "inline-flex items-center shrink-0"
}, /*#__PURE__*/React.createElement("i", {
"data-lucide": "music-2",
className: "w-3 h-3"
})), " Compose"), /*#__PURE__*/React.createElement("button", {
onClick: () => {
setAiPrompt('');
setAiActionLog([]);
File diff suppressed because one or more lines are too long
+52
View File
@@ -121,6 +121,57 @@ const AIGateway = (function() {
}
};
// ai_midi_rearrange_specification.md §3 — Function Tool Schema hỗ trợ 2 mode:
// SIMILAR_VARIATION (biến tấu cùng độ dài) / EXTEND_CONTINUATION (viết tiếp
// các bar sau). AI trả gói dữ liệu có vị trí target trên Timeline.
const REARRANGE_EXTEND_TOOL_SPEC = {
type: 'function',
function: {
name: 'rearrange_or_extend_midi_melody',
description: 'Analyzes source MIDI melody data and returns either a variation (Variation) or continuation (Extend) based on user instructions.',
parameters: {
type: 'object',
properties: {
mode: {
type: 'string',
enum: ['SIMILAR_VARIATION', 'EXTEND_CONTINUATION'],
description: "Mode: 'SIMILAR_VARIATION' (new arrangement of equal length) or 'EXTEND_CONTINUATION' (writes subsequent bars)."
},
composition_title: {
type: 'string',
description: 'Short title describing the new melody style (e.g., Jazz Swing Variation, Epic Extension Part 2)'
},
target_start_bar: {
type: 'number',
description: 'Starting bar number for the generated notes on the Timeline'
},
target_duration_bars: {
type: 'number',
description: 'Total bar duration covered by the generated sequence'
},
soundfont_id: { type: 'string', default: 'generaluser_gs' },
soundfont_bank: { type: 'integer', default: 0 },
soundfont_program: { type: 'integer', default: 0 },
generated_notes: {
type: 'array',
description: 'Array of AI-generated MIDI notes.',
items: {
type: 'object',
properties: {
pitch: { type: 'integer', minimum: 0, maximum: 127 },
start_beat: { type: 'number', description: 'Starting beat position relative to beat 0.0 of the generated item' },
duration_beats: { type: 'number', minimum: 0.1 },
velocity: { type: 'number', minimum: 0.0, maximum: 1.0 }
},
required: ['pitch', 'start_beat', 'duration_beats', 'velocity']
}
}
},
required: ['mode', 'composition_title', 'target_start_bar', 'target_duration_bars', 'generated_notes']
}
}
};
const REARRANGE_SCENARIOS = [
{
id: 'arpeggio',
@@ -456,6 +507,7 @@ Ví dụ: "Hãy chọn và copy từ bar 4 đến bar 12 của track 1 sau đó
return {
DEFAULT_TOOLS,
REARRANGE_TOOL_SPEC,
REARRANGE_EXTEND_TOOL_SPEC,
REARRANGE_SCENARIOS,
detectRearrangeScenario,
buildRearrangeMessage,
+2 -2
View File
@@ -17,14 +17,14 @@
<script src="/static/js/services/storage.js?v=202607271016"></script>
<script src="/static/js/services/soundfontStorage.js?v=202607271016"></script>
<script src="/static/js/services/soundfontPlayer.js?v=202608031400"></script>
<script src="/static/js/services/aiGateway.js?v=202607271016"></script>
<script src="/static/js/services/aiGateway.js?v=202608037200"></script>
<script src="/static/js/services/dawCommandDispatcher.js?v=202607271016"></script>
<script src="/static/js/services/pianoRollTabService.js?v=202607272044"></script>
<script src="/static/js/services/ghostNoteExtractor.js?v=202607271727"></script>
<script src="/static/js/services/midiExtractor.js?v=202607281052"></script>
<script src="/static/js/services/promptTemplateManager.js?v=202607281039"></script>
<script src="/static/js/services/undoRedoEngine.js?v=202607290941"></script>
<script src="/static/js/app.precompiled.js?v=202608037000" defer></script>
<script src="/static/js/app.precompiled.js?v=202608037600" defer></script>
<link rel="stylesheet" href="/static/css/styles.css?v=202607271016">
<style>
:root {