feat: add SoundFont inspection engine + AI instrument schema
- SoundFontInspector (sf2utils) scans .sf2, generates full/condensed catalog - GET /api/v1/plugins/soundfonts/catalog with lazy init + cache invalidation - AI tool generate_multitrack_midi now requires soundfont_id/bank/program - Condensed catalog auto-injected into AI system prompt with bank rules - Server render: FluidSynth program_select uses bank/program + channel routing (drums→ch9) - VST3 pedalboard path inserts CC0 bank select + program change before notes - DecentSamplerManager loads .dspreset with CWD fix for relative sample paths - Pianobook render branch in render_engine.py - Client SonicSF: controllerChange, programChange, applyAITrackInstrument - Post-AI track creation applies instrument via applyAITrackInstrument - Background cache rescan on .sf2 upload, frontend re-fetches catalog - libcurl4 + VST3 dirs in Dockerfile
This commit is contained in:
+17
-1
@@ -3351,9 +3351,13 @@ const PluginManagerModal = ({ isOpen, onClose, pluginsData }) => {
|
||||
try {
|
||||
const result = await window.SonicAPI.uploadSoundFont(file);
|
||||
setSfUploadStatus('Uploaded: ' + result.name);
|
||||
// Refresh plugin list
|
||||
// Refresh plugin list and catalog
|
||||
const data = await window.SonicAPI.listPlugins();
|
||||
setLocalData(data);
|
||||
try {
|
||||
const cat = await window.SonicAPI.getSoundfontCatalog();
|
||||
window.__soundfontCatalog = cat;
|
||||
} catch (_) {}
|
||||
} catch (err) {
|
||||
setSfUploadStatus('Error: ' + err.message);
|
||||
}
|
||||
@@ -7027,6 +7031,11 @@ const App = () => {
|
||||
if (active) setSelectedProviderId(active.id);
|
||||
}
|
||||
} catch (e) { }
|
||||
try {
|
||||
window.SonicAPI.getSoundfontCatalog().then(cat => {
|
||||
window.__soundfontCatalog = cat;
|
||||
}).catch(() => {});
|
||||
} catch (e) { }
|
||||
})();
|
||||
}
|
||||
};
|
||||
@@ -13802,6 +13811,13 @@ const App = () => {
|
||||
}))
|
||||
};
|
||||
targetTrack.midiItems = [...(targetTrack.midiItems || []), newMidiItem];
|
||||
if (aiTrack.soundfont_bank !== undefined && aiTrack.soundfont_program !== undefined) {
|
||||
targetTrack.soundfont_bank = aiTrack.soundfont_bank;
|
||||
targetTrack.soundfont_program = aiTrack.soundfont_program;
|
||||
if (window.SonicSF && window.SonicSF.applyAITrackInstrument) {
|
||||
window.SonicSF.applyAITrackInstrument(aiTrack.soundfont_bank, aiTrack.soundfont_program);
|
||||
}
|
||||
}
|
||||
});
|
||||
return updatedTracks;
|
||||
});
|
||||
|
||||
@@ -48,7 +48,7 @@ const AIGateway = (function() {
|
||||
name: 'fade_out', description: 'Fade-out clip (0.5s đến max)', parameters: { type: 'object', properties: { track_id: { type: 'string' }, duration_seconds: { type: 'number' }, clip_index: { type: 'number', description: 'Chỉ số của clip trên track (1-based, ví dụ: 1 cho clip 1, 2 cho clip 2)' }, clip_id: { type: 'string', description: 'ID của clip cụ thể' } } }
|
||||
}, {
|
||||
name: 'generate_multitrack_midi',
|
||||
description: 'Generates multi-track MIDI data based on genre, bar duration, and requested instruments list.',
|
||||
description: 'Generates multi-track MIDI data along with SoundFont Program configurations for each track.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -57,12 +57,15 @@ const AIGateway = (function() {
|
||||
total_bars: { type: 'integer' },
|
||||
tracks: {
|
||||
type: 'array',
|
||||
description: 'Array of instrument tracks along with their corresponding MIDI notes',
|
||||
description: 'Array of instrument tracks with MIDI notes and SoundFont instrument selection',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
track_name: { type: 'string', description: 'Track name (e.g., String Ensemble, Epic Brass, Taiko Drums)' },
|
||||
instrument_type: { type: 'string', enum: ['STRINGS', 'BRASS', 'SYNTH', 'PERCUSSION', 'DRUMS'] },
|
||||
soundfont_id: { type: 'string', description: "ID of the SoundFont to use (e.g. 'generaluser_gs')" },
|
||||
soundfont_bank: { type: 'integer', default: 0, description: 'MIDI Bank code: 0 for melodic instruments, 128 for Drums/Percussion' },
|
||||
soundfont_program: { type: 'integer', description: 'MIDI Program Number 0-127 matching the instrument name in the SoundFont catalog' },
|
||||
notes: {
|
||||
type: 'array',
|
||||
items: {
|
||||
@@ -77,7 +80,7 @@ const AIGateway = (function() {
|
||||
}
|
||||
}
|
||||
},
|
||||
required: ['track_name', 'instrument_type', 'notes']
|
||||
required: ['track_name', 'instrument_type', 'soundfont_id', 'soundfont_bank', 'soundfont_program', 'notes']
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -176,13 +179,29 @@ const AIGateway = (function() {
|
||||
return calls;
|
||||
}
|
||||
|
||||
function buildCatalogPromptSection() {
|
||||
const catalog = window.__soundfontCatalog;
|
||||
if (!catalog || !catalog.condensed_catalog) return '';
|
||||
const lines = [];
|
||||
for (const [sfId, info] of Object.entries(catalog.condensed_catalog)) {
|
||||
lines.push(`SoundFont ID: '${sfId}' (File: ${info.filename}):`);
|
||||
for (const inst of info.instruments || []) {
|
||||
lines.push(` - ${inst.name}: bank=${inst.bank}, program=${inst.program}`);
|
||||
}
|
||||
}
|
||||
if (lines.length === 0) return '';
|
||||
return `\n=== SOUNDFONT INSTRUMENT CATALOG ===\nYou have the following SoundFont instruments available on the system:\n${lines.join('\n')}\n\nMANDATORY RULES WHEN CREATING TRACKS WITH generate_multitrack_midi:\n1. You MUST look up the catalog above and fill in the correct soundfont_id, soundfont_bank, and soundfont_program for each track.\n2. Melodic instruments (Piano, Strings, Brass, etc.) MUST use soundfont_bank=0.\n3. Drums and Percussion MUST use soundfont_bank=128.\n4. Example: For \"Brass horns\", use soundfont_id="generaluser_gs", soundfont_bank=0, soundfont_program=56.\n5. Example: For \"Drum kit\", use soundfont_id="generaluser_gs", soundfont_bank=128, soundfont_program=0.\n`;
|
||||
}
|
||||
|
||||
function buildUserMessage(prompt, context, systemInstruction = '') {
|
||||
const contextStr = JSON.stringify(context, null, 2);
|
||||
const toolNames = DEFAULT_TOOLS.map(t => ` - ${t.name}: ${t.description}`).join('\n');
|
||||
const catalogSection = buildCatalogPromptSection();
|
||||
return [
|
||||
{ role: 'system', content: `Bạn là trợ lý điều khiển DAW chuyên nghiệp.
|
||||
Nhiệm vụ của bạn là phân tích yêu cầu của người dùng và chuyển đổi thành danh sách các function calls tương ứng.
|
||||
${systemInstruction ? `\nHướng dẫn tạo nhạc đặc biệt từ Preset:\n${systemInstruction}\n` : ''}
|
||||
${catalogSection}
|
||||
QUAN TRỌNG:
|
||||
1. Bạn đang hoạt động ở chế độ một lượt (one-shot). Hãy trả về TẤT CẢ các function calls cần thiết để thực hiện toàn bộ các bước trong yêu cầu của người dùng trong một phản hồi duy nhất. Đừng thực hiện từng bước qua nhiều lượt chat.
|
||||
2. Có thể gọi nhiều function cùng một lúc (gọi song song/nối tiếp). Chúng sẽ được thực thi theo thứ tự bạn trả về.
|
||||
|
||||
@@ -63,6 +63,7 @@ window.API_BASE_URL = window.API_BASE_URL || window.location.origin;
|
||||
savePreferences: (prefs) => apiRequest('/api/v1/user/preferences', { method: 'POST', body: JSON.stringify({ preferences: prefs }) }),
|
||||
|
||||
listPlugins: () => apiRequest('/api/v1/plugins/available', { method: 'GET' }),
|
||||
getSoundfontCatalog: () => apiRequest('/api/v1/plugins/soundfonts/catalog', { method: 'GET' }),
|
||||
listDefaultSoundfonts: () => apiRequest('/api/v1/plugins/default-soundfonts', { method: 'GET' }),
|
||||
listSoundfontInstruments: (sfId) => apiRequest(`/api/v1/plugins/soundfont-instruments/${sfId}`, { method: 'GET' }),
|
||||
renderProject: (projectJson, outputFilename) => apiRequest('/api/v1/plugins/render', { method: 'POST', body: JSON.stringify({ project_json: projectJson, output_filename: outputFilename }) }),
|
||||
|
||||
@@ -30,9 +30,45 @@
|
||||
return window.__sharedAudioCtx;
|
||||
};
|
||||
|
||||
// ── Per-channel MIDI state (16 GM channels) ──
|
||||
const _channels = Array.from({ length: 16 }, () => ({ bank: 0, program: 0, isPercussion: false }));
|
||||
let _nextMelodicChannel = 0;
|
||||
|
||||
const SonicSF = {
|
||||
loadedFonts: {},
|
||||
|
||||
controllerChange: function (channel, controller, value) {
|
||||
if (channel < 0 || channel > 15) return;
|
||||
if (controller === 0) {
|
||||
_channels[channel].bank = value;
|
||||
_channels[channel].isPercussion = (value === 128);
|
||||
}
|
||||
},
|
||||
|
||||
programChange: function (channel, program) {
|
||||
if (channel < 0 || channel > 15) return;
|
||||
_channels[channel].program = program;
|
||||
},
|
||||
|
||||
allocateChannel: function (bank) {
|
||||
if (bank === 128) return 9;
|
||||
const ch = _nextMelodicChannel % 9;
|
||||
_nextMelodicChannel = (_nextMelodicChannel + 1) % 9;
|
||||
return ch;
|
||||
},
|
||||
|
||||
applyAITrackInstrument: function (bank, program) {
|
||||
const channel = this.allocateChannel(bank);
|
||||
this.controllerChange(channel, 0, bank);
|
||||
this.programChange(channel, program);
|
||||
return channel;
|
||||
},
|
||||
|
||||
getChannelState: function (channel) {
|
||||
if (channel < 0 || channel > 15) return null;
|
||||
return { ..._channels[channel] };
|
||||
},
|
||||
|
||||
// Load SoundFont from URL → ArrayBuffer → store in memory
|
||||
loadSoundFont: async function (url) {
|
||||
if (this.loadedFonts[url]) return this.loadedFonts[url];
|
||||
@@ -43,7 +79,7 @@
|
||||
return buffer;
|
||||
},
|
||||
|
||||
playNote: function (note, velocity, durationMs, startTime, program, destinationNode) {
|
||||
playNote: function (note, velocity, durationMs, startTime, program, destinationNode, channel) {
|
||||
const ctx = getCtx();
|
||||
const freq = 440 * Math.pow(2, (note - 69) / 12);
|
||||
if (freq <= 0 || isNaN(freq)) return null;
|
||||
@@ -59,7 +95,11 @@
|
||||
let releaseTime = 0.2;
|
||||
let volFactor = 0.25;
|
||||
|
||||
const prog = program !== undefined ? parseInt(program) : 0;
|
||||
let prog = program !== undefined ? parseInt(program) : 0;
|
||||
if (channel !== undefined && channel >= 0 && channel < 16) {
|
||||
const chState = _channels[channel];
|
||||
prog = chState.program || prog;
|
||||
}
|
||||
if (prog >= 0 && prog <= 7) { // Pianos
|
||||
oscType = 'sine';
|
||||
decayTime = 0.3;
|
||||
|
||||
Reference in New Issue
Block a user