472 lines
27 KiB
JavaScript
472 lines
27 KiB
JavaScript
// SonicForge Studio - AI Gateway & Function Routing
|
|
// LLM Gateway with Function Calling / Structured Outputs (28_AI_PANEL.md §2)
|
|
|
|
const AIGateway = (function() {
|
|
const DEFAULT_TOOLS = [{
|
|
name: 'set_selection', description: 'Chọn vùng timeline', parameters: { type: 'object', properties: { start_bar: { type: 'number' }, end_bar: { type: 'number' }, start_time: { type: 'number' }, end_time: { type: 'number' }, length_bars: { type: 'number' } } }
|
|
}, {
|
|
name: 'cut_audio', description: 'Cắt audio, snap zero-crossing, tạo track mới', parameters: { type: 'object', properties: { track_id: { type: 'string' }, start_time: { type: 'number' }, end_time: { type: 'number' }, start_bar: { type: 'number' }, end_bar: { type: 'number' }, length_bars: { type: 'number' }, snap_silence: { type: 'boolean' }, new_track_name: { type: 'string' } } }
|
|
}, {
|
|
name: 'create_track', description: 'Tạo track mới', parameters: { type: 'object', properties: { name: { type: 'string' }, type: { type: 'string', enum: ['audio', 'midi'] } }, required: ['name'] }
|
|
}, {
|
|
name: 'delete_track', description: 'Xóa track', parameters: { type: 'object', properties: { track_id: { type: 'string' } } }
|
|
}, {
|
|
name: 'rename_track', description: 'Đổi tên track', parameters: { type: 'object', properties: { track_id: { type: 'string' }, name: { type: 'string' } }, required: ['track_id', 'name'] }
|
|
}, {
|
|
name: 'add_clip', description: 'Thêm clip rỗng vào track', parameters: { type: 'object', properties: { track_id: { type: 'string' }, start_time: { type: 'number' }, duration_seconds: { type: 'number' }, start_bar: { type: 'number' }, length_bars: { type: 'number' }, name: { type: 'string' } } }
|
|
}, {
|
|
name: 'remove_clip', description: 'Xóa clip khỏi track', parameters: { type: 'object', properties: { track_id: { type: 'string' }, clip_id: { type: 'string' } }, required: ['clip_id'] }
|
|
}, {
|
|
name: 'set_track_volume', description: 'Chỉnh âm lượng dB', parameters: { type: 'object', properties: { track_id: { type: 'string' }, volume_db: { type: 'number' } }, required: ['volume_db'] }
|
|
}, {
|
|
name: 'set_track_pan', description: 'Chỉnh pan trái/phải', parameters: { type: 'object', properties: { track_id: { type: 'string' }, pan: { type: 'integer' } }, required: ['pan'] }
|
|
}, {
|
|
name: 'toggle_mute', description: 'Mute/unmute track', parameters: { type: 'object', properties: { track_id: { type: 'string' } } }
|
|
}, {
|
|
name: 'toggle_solo', description: 'Solo/unsolo track', parameters: { type: 'object', properties: { track_id: { type: 'string' } } }
|
|
}, {
|
|
name: 'set_bpm', description: 'Thay đổi BPM', parameters: { type: 'object', properties: { bpm: { type: 'number' } }, required: ['bpm'] }
|
|
}, {
|
|
name: 'set_playhead', description: 'Di chuyển playhead', parameters: { type: 'object', properties: { time: { type: 'number' }, bar: { type: 'number' } } }
|
|
}, {
|
|
name: 'add_marker', description: 'Thêm marker', parameters: { type: 'object', properties: { track_id: { type: 'string' }, time: { type: 'number' }, label: { type: 'string' } } }
|
|
}, {
|
|
name: 'process_audio_dsp', description: 'Xử lý DSP: normalize/invert/gain/pitch', parameters: { type: 'object', properties: { track_id: { type: 'string' }, action: { type: 'string', enum: ['normalize', 'invert_phase', 'gain', 'pitch_shift'] }, params: { type: 'object' } }, required: ['track_id', 'action'] }
|
|
}, {
|
|
name: 'create_midi_item', description: 'Tạo MIDI item trên track', parameters: { type: 'object', properties: { track_id: { type: 'string' }, start_bar: { type: 'number' }, length_bars: { type: 'number' } }, required: ['track_id', 'start_bar', 'length_bars'] }
|
|
}, {
|
|
name: 'modify_midi_notes', description: 'Sửa note MIDI trong item', parameters: { type: 'object', properties: { item_id: { type: 'string' }, notes: { type: 'array', items: { type: 'object', properties: { pitch: { type: 'string' }, start_time: { type: 'number' }, duration: { type: 'number' }, velocity: { type: 'integer', minimum: 0, maximum: 127 } }, required: ['pitch', 'start_time', 'duration'] } } }, required: ['item_id', 'notes'] }
|
|
}, {
|
|
name: 'select_item', description: 'Chọn clip/item theo tên', parameters: { type: 'object', properties: { track_id: { type: 'string' }, item_name: { type: 'string' }, select_all: { type: 'boolean' } } }
|
|
}, {
|
|
name: 'scan_track', description: 'Phân tích track: BPM, SR, kênh', parameters: { type: 'object', properties: { track_id: { type: 'string' }, set_tempo: { type: 'boolean' } } }
|
|
}, {
|
|
name: 'fade_in', description: 'Fade-in clip (0.5s đến max)', parameters: { type: 'object', properties: { track_id: { type: 'string' }, duration_seconds: { type: 'number' }, clip_index: { type: 'number', description: 'Chỉ số của clip trên track (1-based, ví dụ: 1 cho clip 1, 2 cho clip 2)' }, clip_id: { type: 'string', description: 'ID của clip cụ thể' } } }
|
|
}, {
|
|
name: 'export_audio', description: 'Xuất file WAV/MP3/OGG và tải về', parameters: { type: 'object', properties: { track_id: { type: 'string' }, format: { type: 'string', enum: ['wav', 'mp3', 'ogg'] }, sample_rate: { type: 'string', enum: ['22500', '44100'] }, bit_depth: { type: 'string', enum: ['8', '16', '24'] }, quality: { type: 'string', enum: ['44khz', 'lossless'] }, channels: { type: 'string', enum: ['mono', 'stereo'] }, start_time: { type: 'number' }, end_time: { type: 'number' }, start_bar: { type: 'number' }, length_bars: { type: 'number' } }, required: ['format'] }
|
|
}, {
|
|
name: 'fade_out', description: 'Fade-out clip (0.5s đến max)', parameters: { type: 'object', properties: { track_id: { type: 'string' }, duration_seconds: { type: 'number' }, clip_index: { type: 'number', description: 'Chỉ số của clip trên track (1-based, ví dụ: 1 cho clip 1, 2 cho clip 2)' }, clip_id: { type: 'string', description: 'ID của clip cụ thể' } } }
|
|
}, {
|
|
name: 'generate_multitrack_midi',
|
|
description: 'Generates multi-track MIDI data along with SoundFont Program configurations for each track.',
|
|
parameters: {
|
|
type: 'object',
|
|
properties: {
|
|
composition_title: { type: 'string', description: 'Title of the musical piece (e.g., Epic Orchestra Intro 8-Bars)' },
|
|
bpm: { type: 'integer' },
|
|
total_bars: { type: 'integer', description: 'Total length of the composition in bars. You MUST populate all bars with notes.' },
|
|
tracks: {
|
|
type: 'array',
|
|
description: 'CRITICAL: Array of instrument tracks. You MUST generate exactly the number of tracks requested by the user. Every track in this array MUST contain a full sequence of notes that spans the entire duration of the piece (from start_beat 0.0 to total_bars * 4.0).',
|
|
items: {
|
|
type: 'object',
|
|
properties: {
|
|
track_name: { type: 'string', description: 'Track name (e.g., String Ensemble, Epic Brass, Taiko Drums)' },
|
|
instrument_type: { type: 'string', enum: ['STRINGS', 'BRASS', 'SYNTH', 'PERCUSSION', 'DRUMS'] },
|
|
soundfont_id: { type: 'string', description: "ID of the SoundFont to use (e.g. 'generaluser_gs')" },
|
|
soundfont_bank: { type: 'integer', default: 0, description: 'MIDI Bank code: 0 for melodic instruments, 128 for Drums/Percussion' },
|
|
soundfont_program: { type: 'integer', description: 'MIDI Program Number 0-127 matching the instrument name in the SoundFont catalog' },
|
|
notes: {
|
|
type: 'array',
|
|
description: 'CRITICAL: Array of MIDI notes. You MUST write notes completely filling all bars from bar 0 (beat 0.0) up to the final bar (beat total_bars * 4.0). Do NOT leave empty bars or stop early. Fill the entire duration of the composition with continuous musical notes.',
|
|
items: {
|
|
type: 'object',
|
|
properties: {
|
|
pitch: { type: 'integer', description: 'MIDI note pitch from 0 to 127 (e.g., C4 = 60, C3 = 48)' },
|
|
start_beat: { type: 'number', description: 'Note start position in beats (from 0.0 to total_bars * 4.0)' },
|
|
duration_beats: { type: 'number', description: 'Note length in beats (e.g., Quarter note = 1.0, Eighth note = 0.5)' },
|
|
velocity: { type: 'number', description: 'Keypress velocity intensity from 0.0 to 1.0' }
|
|
},
|
|
required: ['pitch', 'start_beat', 'duration_beats', 'velocity']
|
|
}
|
|
}
|
|
},
|
|
required: ['track_name', 'instrument_type', 'soundfont_id', 'soundfont_bank', 'soundfont_program', 'notes']
|
|
}
|
|
}
|
|
},
|
|
required: ['composition_title', 'bpm', 'total_bars', 'tracks']
|
|
}
|
|
}];
|
|
|
|
const REARRANGE_TOOL_SPEC = {
|
|
type: 'function',
|
|
function: {
|
|
name: 'rearrange_midi_melody',
|
|
description: 'Accepts source MIDI notes and rearranges/re-harmonizes them into a new musical variation while preserving the core melody. Use this when the user asks to rearrange, remix, or create variations of an existing MIDI item.',
|
|
parameters: {
|
|
type: 'object',
|
|
properties: {
|
|
rearrange_title: { type: 'string', description: 'Title/description of the rearranged version (e.g. "Jazz Variation", "Dark Orchestral Remix")' },
|
|
soundfont_id: { type: 'string', default: 'generaluser_gs' },
|
|
soundfont_bank: { type: 'integer', default: 0 },
|
|
soundfont_program: { type: 'integer', default: 0 },
|
|
rearranged_notes: {
|
|
type: 'array',
|
|
description: 'Array of rearranged MIDI notes. Keep the same total_duration_beats as the original unless user explicitly requests length change.',
|
|
items: {
|
|
type: 'object',
|
|
properties: {
|
|
pitch: { type: 'integer', description: 'MIDI note pitch 0-127 (C4=60)' },
|
|
start_beat: { type: 'number', description: 'Note start position in beats from 0.0' },
|
|
duration_beats: { type: 'number', description: 'Note length in beats (quarter=1.0)' },
|
|
velocity: { type: 'number', description: 'Velocity 0.0-1.0' }
|
|
},
|
|
required: ['pitch', 'start_beat', 'duration_beats', 'velocity']
|
|
}
|
|
}
|
|
},
|
|
required: ['rearrange_title', 'rearranged_notes']
|
|
}
|
|
}
|
|
};
|
|
|
|
const REARRANGE_SCENARIOS = [
|
|
{
|
|
id: 'arpeggio',
|
|
keywords: ['arpeggio', 'arp', 'broken chord', 'broken chords', 'shimmering sequence'],
|
|
description: 'Arpeggio variation',
|
|
technique: 'Subdivide sustained chord notes (longer than 0.5 beats) into sequential 0.25-beat arpeggiated interval steps using octave/triad jumps. Create a driving, shimmering sequence.'
|
|
},
|
|
{
|
|
id: 'harmonies',
|
|
keywords: ['harmony', 'harmonize', 'harmonies', '3rd', '3rds', 'duet', 'chord voicing', 'voicing'],
|
|
description: 'Add parallel harmonies',
|
|
technique: 'For each source pitch P, insert parallel harmonizing notes at pitch P+3 or P+4 (Major/Minor 3rds) matching the same start_beat. Create richer, fuller texture resembling two instruments playing in duet.'
|
|
},
|
|
{
|
|
id: 'syncopation',
|
|
keywords: ['syncopation', 'syncopate', 'syncopated', 'off-beat', 'off beat', 'funk', 'latin', 'rnb', 'r&b'],
|
|
description: 'Syncopation / off-beat feel',
|
|
technique: 'Shift start_beat alignment off strong beats (0.0, 1.0, 2.0, 3.0) onto off-beats (0.5, 1.5, 2.5, 3.5). Create a rhythmic, syncopated feel.'
|
|
},
|
|
{
|
|
id: 'walking_bass',
|
|
keywords: ['walking bass', 'bassline', 'bass line', 'walking'],
|
|
description: 'Walking bass line',
|
|
technique: 'Lower pitches to bass register (36-48 range). Build continuous quarter-note steps (1.0 beat duration) following the chord outline. Create an improvisational walking bassline.'
|
|
},
|
|
{
|
|
id: 'jazz_swing',
|
|
keywords: ['jazz', 'swing', 'bebop', 'bop', 'jazz swing'],
|
|
description: 'Jazz Swing style',
|
|
technique: 'Apply Jazz Swing characteristics: off-beat syncopation, 7th/9th chord extensions, rhythmic variations, and swung eighth notes (triplet feel). Preserve the core melodic outline while adding jazz harmony.'
|
|
},
|
|
{
|
|
id: 'synthwave',
|
|
keywords: ['synthwave', '80s', 'retro', 'synth wave', 'outrun', 'retrowave'],
|
|
description: 'Synthwave / 80s style',
|
|
technique: 'Transform into 80s Synthwave style: driving 8th-note bass arpeggios, gated reverb snare, analog synth lead with portamento, and pulsating chord pads. Use octave jumps in the bass.'
|
|
},
|
|
{
|
|
id: 'cinematic',
|
|
keywords: ['cinematic', 'orchestral', 'epic', 'film score', 'movie', 'hollywood', 'symphonic'],
|
|
description: 'Cinematic Orchestral style',
|
|
technique: 'Transform into dramatic Cinematic Orchestral style: swelling dynamics, brass stabs on downbeats, string ostinato patterns, taiko percussion hits on strong beats. Layer multiple octaves for epic width.'
|
|
},
|
|
{
|
|
id: 'simplify',
|
|
keywords: ['simplify', 'minimal', 'strip', 'downbeat', 'reduce', 'sparse'],
|
|
description: 'Simplify / strip down',
|
|
technique: 'Keep only the downbeat notes (start_beat at 0.0, 1.0, 2.0, 3.0, etc.). Strip out embellishments, passing tones, and grace notes. Reduce chord voicings to root and 5th only.'
|
|
},
|
|
{
|
|
id: 'passing_tones',
|
|
keywords: ['passing tone', 'chromatic', 'chromaticism', 'smooth', 'approach note'],
|
|
description: 'Add passing tones / chromaticism',
|
|
technique: 'Insert passing tones and chromatic approach notes between chord tones to smooth out the melodic progression. Use half-step and whole-step approach notes targeting chord tones on strong beats.'
|
|
}
|
|
];
|
|
|
|
function detectRearrangeScenario(prompt) {
|
|
if (!prompt) return null;
|
|
const lower = prompt.toLowerCase();
|
|
const matched = [];
|
|
for (const sc of REARRANGE_SCENARIOS) {
|
|
const hits = sc.keywords.filter(kw => lower.includes(kw.toLowerCase()));
|
|
if (hits.length > 0) {
|
|
matched.push({ scenario: sc, hitCount: hits.length });
|
|
}
|
|
}
|
|
matched.sort((a, b) => b.hitCount - a.hitCount);
|
|
return matched.length > 0 ? matched[0].scenario : null;
|
|
}
|
|
|
|
function buildRearrangeMessage(prompt, sourceContext) {
|
|
const notesJson = JSON.stringify(sourceContext.notes, null, 2);
|
|
const scenario = detectRearrangeScenario(prompt);
|
|
const techniqueSection = scenario
|
|
? `\nDETECTED SCENARIO: ${scenario.description}\nSPECIFIC TECHNIQUE REQUIRED: ${scenario.technique}\n`
|
|
: '';
|
|
|
|
const rules = [
|
|
'1. Use the rearrange_midi_melody tool to return the rearranged notes.',
|
|
'2. PRESERVE the core melodic outline and overall structure unless the user explicitly asks for a complete transformation.',
|
|
'3. Keep the total duration (' + sourceContext.total_beats + ' beats) the same unless user requests a different length.',
|
|
'4. The rearranged_notes array MUST contain notes with pitch (0-127), start_beat (0.0 to ' + sourceContext.total_beats + '.0), duration_beats, and velocity (0.0-1.0).',
|
|
'5. Start beats should remain within 0-' + sourceContext.total_beats + ' range.',
|
|
'6. You may add, remove, or modify notes to achieve the requested style.',
|
|
'7. ALWAYS return a rearrange_title describing what was created (e.g. "Jazz Swing Variation of Piano Lead").'
|
|
];
|
|
|
|
return [
|
|
{ role: 'system', content: `You are a professional Music Composer & Arranger.
|
|
|
|
SOURCE MIDI CONTEXT:
|
|
- Track: ${sourceContext.track_name}
|
|
- Item: ${sourceContext.item_name}
|
|
- Duration: ${sourceContext.duration_bars} bars (${sourceContext.total_beats} beats)
|
|
- BPM: ${sourceContext.bpm}
|
|
- Time Signature: 4/4
|
|
- Total notes: ${sourceContext.total_notes}
|
|
|
|
ORIGINAL MELODY NOTES (pitch, note_name, start_beat, duration_beats, velocity):
|
|
${notesJson}
|
|
${techniqueSection}
|
|
STRICT REARRANGE RULES:
|
|
${rules.join('\n')}` },
|
|
{ role: 'user', content: `${prompt}\n\nRearrange the source MIDI notes above according to this request. Return the result via the rearrange_midi_melody tool.` }
|
|
];
|
|
}
|
|
|
|
function parseOrigin(urlStr) {
|
|
try { const u = new URL(urlStr); return `${u.protocol}//${u.hostname}${u.port ? ':'+u.port : ''}`; } catch (_) { return null; }
|
|
}
|
|
|
|
function isLocalhost(urlStr) {
|
|
try {
|
|
const u = new URL(urlStr);
|
|
return u.hostname === 'localhost' || u.hostname === '127.0.0.1' || u.hostname === '0.0.0.0' || u.hostname === '::1';
|
|
} catch (_) { return false; }
|
|
}
|
|
|
|
async function callLLM({ provider, model, apiKey, baseUrl, messages, tools, toolChoice }) {
|
|
const base = baseUrl.replace(/\/$/, '');
|
|
const url = `${base}/chat/completions`;
|
|
const origin = window.location.origin;
|
|
const urlOrigin = parseOrigin(url);
|
|
const appOrigin = parseOrigin(origin);
|
|
const sameOrigin = urlOrigin === appOrigin;
|
|
const targetIsLocal = isLocalhost(url);
|
|
|
|
const headers = {
|
|
'Content-Type': 'application/json',
|
|
...(apiKey ? { 'Authorization': `Bearer ${apiKey}` } : {})
|
|
};
|
|
|
|
const body = {
|
|
model,
|
|
messages,
|
|
stream: false,
|
|
...(tools && tools.length > 0 ? { tools: tools.map(t => ({ type: 'function', function: t })) } : {}),
|
|
...(toolChoice ? { tool_choice: toolChoice } : {})
|
|
};
|
|
|
|
let response;
|
|
|
|
if (sameOrigin) {
|
|
response = await fetch(url, {
|
|
method: 'POST',
|
|
headers,
|
|
body: JSON.stringify(body)
|
|
});
|
|
} else if (targetIsLocal && !isLocalhost(origin)) {
|
|
throw new Error(`AI provider local (${url}) không khả dụng từ domain từ xa (${origin}).\nHãy dùng provider từ xa (OpenAI, Anthropic...) hoặc dùng CORS plugin trình duyệt.`);
|
|
} else {
|
|
response = await fetch(`${origin}/api/v1/ai/proxy`, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ url, headers, body })
|
|
});
|
|
}
|
|
|
|
if (!response.ok) {
|
|
const errText = await response.text();
|
|
let detail = errText;
|
|
try { const j = JSON.parse(errText); if (j.detail) detail = j.detail; } catch (_) {}
|
|
throw new Error(detail);
|
|
}
|
|
|
|
return await response.json();
|
|
}
|
|
|
|
function extractFunctionCalls(completion) {
|
|
const calls = [];
|
|
const choice = completion.choices && completion.choices[0];
|
|
if (!choice) return calls;
|
|
|
|
const msg = choice.message;
|
|
if (msg.tool_calls && Array.isArray(msg.tool_calls)) {
|
|
for (const tc of msg.tool_calls) {
|
|
if (tc.type === 'function' && tc.function) {
|
|
let args = {};
|
|
try { args = JSON.parse(tc.function.arguments || '{}'); } catch (e) { args = { raw: tc.function.arguments }; }
|
|
calls.push({
|
|
id: tc.id,
|
|
name: tc.function.name,
|
|
arguments: args
|
|
});
|
|
}
|
|
}
|
|
} else if (msg.function_call) {
|
|
let args = {};
|
|
try { args = JSON.parse(msg.function_call.arguments || '{}'); } catch (e) { args = { raw: msg.function_call.arguments }; }
|
|
calls.push({
|
|
id: 'call_' + Date.now(),
|
|
name: msg.function_call.name,
|
|
arguments: args
|
|
});
|
|
}
|
|
return calls;
|
|
}
|
|
|
|
function buildCatalogPromptSection() {
|
|
const catalog = window.__soundfontCatalog;
|
|
if (!catalog || !catalog.condensed_catalog) return '';
|
|
const lines = [];
|
|
for (const [sfId, info] of Object.entries(catalog.condensed_catalog)) {
|
|
lines.push(`SoundFont ID: '${sfId}' (File: ${info.filename}):`);
|
|
for (const inst of info.instruments || []) {
|
|
lines.push(` - ${inst.name}: bank=${inst.bank}, program=${inst.program}`);
|
|
}
|
|
}
|
|
if (lines.length === 0) return '';
|
|
return `\n=== SOUNDFONT INSTRUMENT CATALOG ===\nYou have the following SoundFont instruments available on the system:\n${lines.join('\n')}\n\nMANDATORY RULES WHEN CREATING TRACKS WITH generate_multitrack_midi:\n1. You MUST look up the catalog above and fill in the correct soundfont_id, soundfont_bank, and soundfont_program for each track.\n2. Melodic instruments (Piano, Strings, Brass, etc.) MUST use soundfont_bank=0.\n3. Drums and Percussion MUST use soundfont_bank=128.\n4. Example: For \"Brass horns\", use soundfont_id="generaluser_gs", soundfont_bank=0, soundfont_program=56.\n5. Example: For \"Drum kit\", use soundfont_id="generaluser_gs", soundfont_bank=128, soundfont_program=0.\n`;
|
|
}
|
|
|
|
function buildUserMessage(prompt, context, systemInstruction = '') {
|
|
const contextStr = JSON.stringify(context, null, 2);
|
|
const toolNames = DEFAULT_TOOLS.map(t => ` - ${t.name}: ${t.description}`).join('\n');
|
|
const catalogSection = buildCatalogPromptSection();
|
|
return [
|
|
{ role: 'system', content: `Bạn là trợ lý điều khiển DAW chuyên nghiệp.
|
|
Nhiệm vụ của bạn là phân tích yêu cầu của người dùng và chuyển đổi thành danh sách các function calls tương ứng.
|
|
${systemInstruction ? `\nHướng dẫn tạo nhạc đặc biệt từ Preset:\n${systemInstruction}\n` : ''}
|
|
${catalogSection}
|
|
|
|
=== HƯỚNG DẪN SOẠN NHẠC MIDI / MIDI COMPOSITION RULES ===
|
|
KHI NGƯỜI DÙNG YÊU CẦU TẠO NHẠC / COMPOSITION RULES:
|
|
1. FULL TRACKS & BARS: If the user requests X tracks and Y bars, you MUST generate exactly X tracks. Each track MUST contain a continuous sequence of MIDI notes starting from beat 0.0 and stretching all the way to beat Y * 4.0 (the end of the composition).
|
|
2. NO EARLY STOPPING: Do NOT stop early or leave empty bars at the end or in the middle. Every track must be fully populated with notes throughout the entire duration.
|
|
3. EXPRESS MELODY & EMOTION: The generated MIDI notes (pitch, start_beat, duration_beats, velocity) must express the requested musical emotion (e.g., happy, sad, epic, energetic, melancholic). Use rich harmonies/chords for backing tracks (Strings, Pads, Piano) and expressive, rhythmic melodies for Lead/Solo tracks. Do NOT write single repeating notes or overly sparse patterns unless explicitly asked.
|
|
4. VIẾT ĐẦY ĐỦ CÁC NOTE: Bạn phải viết đầy đủ các note cho TẤT CẢ các tracks được yêu cầu, và trải dài trong SUỐT chiều dài số bars yêu cầu (ví dụ: yêu cầu 8 bars và 6 tracks thì phải tạo đủ 6 tracks, mỗi track phải có các note MIDI bắt đầu từ beat 0.0 kéo dài liên tục đến beat 32.0 (8 bars * 4 beat/bar)).
|
|
|
|
QUAN TRỌNG:
|
|
1. Bạn đang hoạt động ở chế độ một lượt (one-shot). Hãy trả về TẤT CẢ các function calls cần thiết để thực hiện toàn bộ các bước trong yêu cầu của người dùng trong một phản hồi duy nhất. Đừng thực hiện từng bước qua nhiều lượt chat.
|
|
2. Có thể gọi nhiều function cùng một lúc (gọi song song/nối tiếp). Chúng sẽ được thực thi theo thứ tự bạn trả về.
|
|
3. Khi người dùng yêu cầu chọn và cắt/sao chép/copy một đoạn nhạc từ track cũ để tạo đoạn nhạc mới (bằng lệnh 'cut_audio'), và sau đó yêu cầu xử lý tiếp đoạn nhạc mới tạo đó (ví dụ: 'sau đó fade in đoạn đó', 'chỉnh âm lượng đoạn đó', 'xuất mp3 đoạn đó'...), thì tất cả các lệnh xử lý tiếp theo này (như 'fade_in', 'export_audio', 'set_track_volume') PHẢI để trống tham số 'track_id' (hoặc truyền null/không truyền) để hệ thống tự động áp dụng lên track mới vừa được tạo ra. KHÔNG ĐƯỢC dùng 'track_id' của track gốc ban đầu cho các lệnh xử lý phía sau.
|
|
Ví dụ: "Hãy chọn và copy từ bar 4 đến bar 12 của track 1 sau đó fade in clip đó 3s, xuất ra mp3" -> Bạn phải trả về đồng thời 3 cuộc gọi hàm theo thứ tự:
|
|
- cut_audio({"track_id": "1", "start_bar": 4, "end_bar": 12})
|
|
- fade_in({"duration_seconds": 3}) (không truyền track_id)
|
|
- export_audio({"format": "mp3"}) (không truyền track_id)
|
|
4. Bar 0 đại diện cho bar đầu tiên trên timeline.` },
|
|
{ role: 'user', content: `Ngữ cảnh DAW hiện tại:\n${contextStr}\n\nYêu cầu người dùng: ${prompt}` }
|
|
];
|
|
}
|
|
|
|
function buildAIPromptContext(dawState) {
|
|
const tracks = (dawState.tracks || []).map(t => {
|
|
const clips = t.clips && t.clips.length > 0 ? t.clips : (t.buffer ? [{ id: 'default_' + t.id, name: t.name, startTime: t.startTime || 0, duration: t.buffer.duration }] : []);
|
|
const se = t.synth_engine || null;
|
|
return {
|
|
id: t.id,
|
|
name: t.name,
|
|
type: t.buffer ? 'audio' : 'empty',
|
|
hasBuffer: !!t.buffer,
|
|
muted: t.muted,
|
|
solo: t.solo,
|
|
volumeDb: t.volumeDb ?? 0,
|
|
pan: t.pan ?? 0,
|
|
synth_engine: se ? { type: se.type, plugin_id: se.plugin_id, soundfont_bank: se.soundfont_bank, soundfont_program: se.soundfont_program, soundfont_id: se.soundfont_id } : undefined,
|
|
clips: clips.map(c => ({ id: c.id, name: c.name, startTime: parseFloat((c.startTime || 0).toFixed(3)), duration: parseFloat((c.buffer ? c.buffer.duration : 0).toFixed(3)) }))
|
|
};
|
|
});
|
|
return {
|
|
tempo: parseInt(dawState.bpm || '120'),
|
|
timeSignature: '4/4',
|
|
selectedTrackId: dawState.selectedTrackId || null,
|
|
playheadPosition: parseFloat((dawState.currentTime || 0).toFixed(3)),
|
|
selection: (dawState.selLeft !== null && dawState.selRight !== null && dawState.selRight > dawState.selLeft) ? {
|
|
start: parseFloat(dawState.selLeft.toFixed(3)),
|
|
end: parseFloat(dawState.selRight.toFixed(3)),
|
|
length: parseFloat((dawState.selRight - dawState.selLeft).toFixed(3))
|
|
} : null,
|
|
tracks
|
|
};
|
|
}
|
|
|
|
async function executeAIPrompt({ prompt, provider, model, apiKey, baseUrl, dawContext, tools, systemInstruction }) {
|
|
const messages = buildUserMessage(prompt, dawContext, systemInstruction);
|
|
const toolList = tools || DEFAULT_TOOLS;
|
|
|
|
const completion = await callLLM({
|
|
provider,
|
|
model,
|
|
apiKey,
|
|
baseUrl,
|
|
messages,
|
|
tools: toolList,
|
|
toolChoice: 'auto'
|
|
});
|
|
|
|
if (completion && completion.error) {
|
|
const errMsg = completion.error.message || completion.error.code || JSON.stringify(completion.error);
|
|
throw new Error(`AI Provider error: ${errMsg}`);
|
|
}
|
|
|
|
const functionCalls = extractFunctionCalls(completion);
|
|
const textResponse = completion.choices && completion.choices[0] && completion.choices[0].message && completion.choices[0].message.content
|
|
? completion.choices[0].message.content
|
|
: '';
|
|
|
|
return {
|
|
functionCalls,
|
|
textResponse,
|
|
raw: completion
|
|
};
|
|
}
|
|
|
|
async function createMidiItem(args) {
|
|
return fetch('/api/audio_editor', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ action: 'add_midi', ...args })
|
|
}).then(r => r.json());
|
|
}
|
|
|
|
async function modifyMidiNotes(args) {
|
|
return fetch('/api/audio_editor', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ action: 'modify_midi_notes', ...args })
|
|
}).then(r => r.json());
|
|
}
|
|
|
|
async function processAIDSP(args) {
|
|
return fetch('/api/ai_dsp_engine', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ action: 'process_ai_dsp', ...args })
|
|
}).then(r => r.json());
|
|
}
|
|
|
|
return {
|
|
DEFAULT_TOOLS,
|
|
REARRANGE_TOOL_SPEC,
|
|
REARRANGE_SCENARIOS,
|
|
detectRearrangeScenario,
|
|
buildRearrangeMessage,
|
|
callLLM,
|
|
extractFunctionCalls,
|
|
buildUserMessage,
|
|
buildAIPromptContext,
|
|
executeAIPrompt,
|
|
createMidiItem,
|
|
modifyMidiNotes,
|
|
processAIDSP
|
|
};
|
|
})();
|
|
|
|
window.executeAIPrompt = AIGateway.executeAIPrompt;
|
|
window.AIGateway = AIGateway;
|