250 lines
12 KiB
JavaScript
250 lines
12 KiB
JavaScript
// SonicForge Studio - AI Gateway & Function Routing
|
|
// LLM Gateway with Function Calling / Structured Outputs (28_AI_PANEL.md §2)
|
|
|
|
const AIGateway = (function() {
|
|
const DEFAULT_TOOLS = [{
|
|
name: 'set_selection', description: 'Chọn vùng timeline', parameters: { type: 'object', properties: { start_bar: { type: 'number' }, end_bar: { type: 'number' }, start_time: { type: 'number' }, end_time: { type: 'number' }, length_bars: { type: 'number' } } }
|
|
}, {
|
|
name: 'cut_audio', description: 'Cắt audio, snap zero-crossing, tạo track mới', parameters: { type: 'object', properties: { track_id: { type: 'string' }, start_time: { type: 'number' }, end_time: { type: 'number' }, start_bar: { type: 'number' }, end_bar: { type: 'number' }, length_bars: { type: 'number' }, snap_silence: { type: 'boolean' }, new_track_name: { type: 'string' } } }
|
|
}, {
|
|
name: 'create_track', description: 'Tạo track mới', parameters: { type: 'object', properties: { name: { type: 'string' }, type: { type: 'string', enum: ['audio', 'midi'] } }, required: ['name'] }
|
|
}, {
|
|
name: 'delete_track', description: 'Xóa track', parameters: { type: 'object', properties: { track_id: { type: 'string' } } }
|
|
}, {
|
|
name: 'rename_track', description: 'Đổi tên track', parameters: { type: 'object', properties: { track_id: { type: 'string' }, name: { type: 'string' } }, required: ['track_id', 'name'] }
|
|
}, {
|
|
name: 'add_clip', description: 'Thêm clip rỗng vào track', parameters: { type: 'object', properties: { track_id: { type: 'string' }, start_time: { type: 'number' }, duration_seconds: { type: 'number' }, start_bar: { type: 'number' }, length_bars: { type: 'number' }, name: { type: 'string' } } }
|
|
}, {
|
|
name: 'remove_clip', description: 'Xóa clip khỏi track', parameters: { type: 'object', properties: { track_id: { type: 'string' }, clip_id: { type: 'string' } }, required: ['clip_id'] }
|
|
}, {
|
|
name: 'set_track_volume', description: 'Chỉnh âm lượng dB', parameters: { type: 'object', properties: { track_id: { type: 'string' }, volume_db: { type: 'number' } }, required: ['volume_db'] }
|
|
}, {
|
|
name: 'set_track_pan', description: 'Chỉnh pan trái/phải', parameters: { type: 'object', properties: { track_id: { type: 'string' }, pan: { type: 'integer' } }, required: ['pan'] }
|
|
}, {
|
|
name: 'toggle_mute', description: 'Mute/unmute track', parameters: { type: 'object', properties: { track_id: { type: 'string' } } }
|
|
}, {
|
|
name: 'toggle_solo', description: 'Solo/unsolo track', parameters: { type: 'object', properties: { track_id: { type: 'string' } } }
|
|
}, {
|
|
name: 'set_bpm', description: 'Thay đổi BPM', parameters: { type: 'object', properties: { bpm: { type: 'number' } }, required: ['bpm'] }
|
|
}, {
|
|
name: 'set_playhead', description: 'Di chuyển playhead', parameters: { type: 'object', properties: { time: { type: 'number' }, bar: { type: 'number' } } }
|
|
}, {
|
|
name: 'add_marker', description: 'Thêm marker', parameters: { type: 'object', properties: { track_id: { type: 'string' }, time: { type: 'number' }, label: { type: 'string' } } }
|
|
}, {
|
|
name: 'process_audio_dsp', description: 'Xử lý DSP: normalize/invert/gain/pitch', parameters: { type: 'object', properties: { track_id: { type: 'string' }, action: { type: 'string', enum: ['normalize', 'invert_phase', 'gain', 'pitch_shift'] }, params: { type: 'object' } }, required: ['track_id', 'action'] }
|
|
}, {
|
|
name: 'create_midi_item', description: 'Tạo MIDI item trên track', parameters: { type: 'object', properties: { track_id: { type: 'string' }, start_bar: { type: 'number' }, length_bars: { type: 'number' } }, required: ['track_id', 'start_bar', 'length_bars'] }
|
|
}, {
|
|
name: 'modify_midi_notes', description: 'Sửa note MIDI trong item', parameters: { type: 'object', properties: { item_id: { type: 'string' }, notes: { type: 'array', items: { type: 'object', properties: { pitch: { type: 'string' }, start_time: { type: 'number' }, duration: { type: 'number' }, velocity: { type: 'integer', minimum: 0, maximum: 127 } }, required: ['pitch', 'start_time', 'duration'] } } }, required: ['item_id', 'notes'] }
|
|
}, {
|
|
name: 'select_item', description: 'Chọn clip/item theo tên', parameters: { type: 'object', properties: { track_id: { type: 'string' }, item_name: { type: 'string' }, select_all: { type: 'boolean' } } }
|
|
}, {
|
|
name: 'scan_track', description: 'Phân tích track: BPM, SR, kênh', parameters: { type: 'object', properties: { track_id: { type: 'string' }, set_tempo: { type: 'boolean' } } }
|
|
}, {
|
|
name: 'fade_in', description: 'Fade-in clip (0.5s đến max)', parameters: { type: 'object', properties: { track_id: { type: 'string' }, duration_seconds: { type: 'number' } } }
|
|
}, {
|
|
name: 'export_audio', description: 'Xuất file WAV/MP3/OGG và tải về', parameters: { type: 'object', properties: { track_id: { type: 'string' }, format: { type: 'string', enum: ['wav', 'mp3', 'ogg'] }, sample_rate: { type: 'string', enum: ['22500', '44100'] }, bit_depth: { type: 'string', enum: ['8', '16', '24'] }, quality: { type: 'string', enum: ['44khz', 'lossless'] }, channels: { type: 'string', enum: ['mono', 'stereo'] }, start_time: { type: 'number' }, end_time: { type: 'number' }, start_bar: { type: 'number' }, length_bars: { type: 'number' } }, required: ['format'] }
|
|
}, {
|
|
name: 'fade_out', description: 'Fade-out clip (0.5s đến max)', parameters: { type: 'object', properties: { track_id: { type: 'string' }, duration_seconds: { type: 'number' } } }
|
|
}];
|
|
|
|
function parseOrigin(urlStr) {
|
|
try { const u = new URL(urlStr); return `${u.protocol}//${u.hostname}${u.port ? ':'+u.port : ''}`; } catch (_) { return null; }
|
|
}
|
|
|
|
function isLocalhost(urlStr) {
|
|
try {
|
|
const u = new URL(urlStr);
|
|
return u.hostname === 'localhost' || u.hostname === '127.0.0.1' || u.hostname === '0.0.0.0' || u.hostname === '::1';
|
|
} catch (_) { return false; }
|
|
}
|
|
|
|
async function callLLM({ provider, model, apiKey, baseUrl, messages, tools, toolChoice }) {
|
|
const base = baseUrl.replace(/\/$/, '');
|
|
const url = `${base}/chat/completions`;
|
|
const origin = window.location.origin;
|
|
const urlOrigin = parseOrigin(url);
|
|
const appOrigin = parseOrigin(origin);
|
|
const sameOrigin = urlOrigin === appOrigin;
|
|
const targetIsLocal = isLocalhost(url);
|
|
|
|
const headers = {
|
|
'Content-Type': 'application/json',
|
|
...(apiKey ? { 'Authorization': `Bearer ${apiKey}` } : {})
|
|
};
|
|
|
|
const body = {
|
|
model,
|
|
messages,
|
|
stream: false,
|
|
...(tools && tools.length > 0 ? { tools: tools.map(t => ({ type: 'function', function: t })) } : {}),
|
|
...(toolChoice ? { tool_choice: toolChoice } : {})
|
|
};
|
|
|
|
let response;
|
|
|
|
if (sameOrigin) {
|
|
response = await fetch(url, {
|
|
method: 'POST',
|
|
headers,
|
|
body: JSON.stringify(body)
|
|
});
|
|
} else if (targetIsLocal && !isLocalhost(origin)) {
|
|
throw new Error(`AI provider local (${url}) không khả dụng từ domain từ xa (${origin}).\nHãy dùng provider từ xa (OpenAI, Anthropic...) hoặc dùng CORS plugin trình duyệt.`);
|
|
} else {
|
|
response = await fetch(`${origin}/api/v1/ai/proxy`, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ url, headers, body })
|
|
});
|
|
}
|
|
|
|
if (!response.ok) {
|
|
const errText = await response.text();
|
|
let detail = errText;
|
|
try { const j = JSON.parse(errText); if (j.detail) detail = j.detail; } catch (_) {}
|
|
throw new Error(detail);
|
|
}
|
|
|
|
return await response.json();
|
|
}
|
|
|
|
function extractFunctionCalls(completion) {
|
|
const calls = [];
|
|
const choice = completion.choices && completion.choices[0];
|
|
if (!choice) return calls;
|
|
|
|
const msg = choice.message;
|
|
if (msg.tool_calls && Array.isArray(msg.tool_calls)) {
|
|
for (const tc of msg.tool_calls) {
|
|
if (tc.type === 'function' && tc.function) {
|
|
let args = {};
|
|
try { args = JSON.parse(tc.function.arguments || '{}'); } catch (e) { args = { raw: tc.function.arguments }; }
|
|
calls.push({
|
|
id: tc.id,
|
|
name: tc.function.name,
|
|
arguments: args
|
|
});
|
|
}
|
|
}
|
|
} else if (msg.function_call) {
|
|
let args = {};
|
|
try { args = JSON.parse(msg.function_call.arguments || '{}'); } catch (e) { args = { raw: msg.function_call.arguments }; }
|
|
calls.push({
|
|
id: 'call_' + Date.now(),
|
|
name: msg.function_call.name,
|
|
arguments: args
|
|
});
|
|
}
|
|
return calls;
|
|
}
|
|
|
|
function buildUserMessage(prompt, context) {
|
|
const contextStr = JSON.stringify(context, null, 2);
|
|
const toolNames = DEFAULT_TOOLS.map(t => ` - ${t.name}: ${t.description}`).join('\n');
|
|
return [
|
|
{ role: 'system', content: `Bạn là trợ lý DAW. Dùng function calls. Bar 0 = bar đầu. Có thể gọi nhiều function cùng lúc.` },
|
|
{ role: 'user', content: `Ngữ cảnh DAW hiện tại:\n${contextStr}\n\nYêu cầu người dùng: ${prompt}` }
|
|
];
|
|
}
|
|
|
|
function buildAIPromptContext(dawState) {
|
|
const tracks = (dawState.tracks || []).map(t => {
|
|
const clips = t.clips && t.clips.length > 0 ? t.clips : (t.buffer ? [{ id: 'default_' + t.id, name: t.name, startTime: t.startTime || 0, duration: t.buffer.duration }] : []);
|
|
return {
|
|
id: t.id,
|
|
name: t.name,
|
|
type: t.buffer ? 'audio' : 'empty',
|
|
hasBuffer: !!t.buffer,
|
|
muted: t.muted,
|
|
solo: t.solo,
|
|
volumeDb: t.volumeDb ?? 0,
|
|
pan: t.pan ?? 0,
|
|
clips: clips.map(c => ({ id: c.id, name: c.name, startTime: parseFloat((c.startTime || 0).toFixed(3)), duration: parseFloat((c.buffer ? c.buffer.duration : 0).toFixed(3)) }))
|
|
};
|
|
});
|
|
return {
|
|
tempo: parseInt(dawState.bpm || '120'),
|
|
timeSignature: '4/4',
|
|
selectedTrackId: dawState.selectedTrackId || null,
|
|
playheadPosition: parseFloat((dawState.currentTime || 0).toFixed(3)),
|
|
selection: (dawState.selLeft !== null && dawState.selRight !== null && dawState.selRight > dawState.selLeft) ? {
|
|
start: parseFloat(dawState.selLeft.toFixed(3)),
|
|
end: parseFloat(dawState.selRight.toFixed(3)),
|
|
length: parseFloat((dawState.selRight - dawState.selLeft).toFixed(3))
|
|
} : null,
|
|
tracks
|
|
};
|
|
}
|
|
|
|
async function executeAIPrompt({ prompt, provider, model, apiKey, baseUrl, dawContext, tools }) {
|
|
const messages = buildUserMessage(prompt, dawContext);
|
|
const toolList = tools || DEFAULT_TOOLS;
|
|
|
|
const completion = await callLLM({
|
|
provider,
|
|
model,
|
|
apiKey,
|
|
baseUrl,
|
|
messages,
|
|
tools: toolList,
|
|
toolChoice: 'auto'
|
|
});
|
|
|
|
if (completion && completion.error) {
|
|
const errMsg = completion.error.message || completion.error.code || JSON.stringify(completion.error);
|
|
throw new Error(`AI Provider error: ${errMsg}`);
|
|
}
|
|
|
|
const functionCalls = extractFunctionCalls(completion);
|
|
const textResponse = completion.choices && completion.choices[0] && completion.choices[0].message && completion.choices[0].message.content
|
|
? completion.choices[0].message.content
|
|
: '';
|
|
|
|
return {
|
|
functionCalls,
|
|
textResponse,
|
|
raw: completion
|
|
};
|
|
}
|
|
|
|
async function createMidiItem(args) {
|
|
return fetch('/api/audio_editor', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ action: 'add_midi', ...args })
|
|
}).then(r => r.json());
|
|
}
|
|
|
|
async function modifyMidiNotes(args) {
|
|
return fetch('/api/audio_editor', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ action: 'modify_midi_notes', ...args })
|
|
}).then(r => r.json());
|
|
}
|
|
|
|
async function processAIDSP(args) {
|
|
return fetch('/api/ai_dsp_engine', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ action: 'process_ai_dsp', ...args })
|
|
}).then(r => r.json());
|
|
}
|
|
|
|
return {
|
|
DEFAULT_TOOLS,
|
|
callLLM,
|
|
extractFunctionCalls,
|
|
buildUserMessage,
|
|
buildAIPromptContext,
|
|
executeAIPrompt,
|
|
createMidiItem,
|
|
modifyMidiNotes,
|
|
processAIDSP
|
|
};
|
|
})();
|
|
|
|
window.executeAIPrompt = AIGateway.executeAIPrompt;
|
|
window.AIGateway = AIGateway;
|