feat: add SoundFont inspection engine + AI instrument schema

- SoundFontInspector (sf2utils) scans .sf2, generates full/condensed catalog
- GET /api/v1/plugins/soundfonts/catalog with lazy init + cache invalidation
- AI tool generate_multitrack_midi now requires soundfont_id/bank/program
- Condensed catalog auto-injected into AI system prompt with bank rules
- Server render: FluidSynth program_select uses bank/program + channel routing (drums→ch9)
- VST3 pedalboard path inserts CC0 bank select + program change before notes
- DecentSamplerManager loads .dspreset with CWD fix for relative sample paths
- Pianobook render branch in render_engine.py
- Client SonicSF: controllerChange, programChange, applyAITrackInstrument
- Post-AI track creation applies instrument via applyAITrackInstrument
- Background cache rescan on .sf2 upload, frontend re-fetches catalog
- libcurl4 + VST3 dirs in Dockerfile
This commit is contained in:
2026-07-26 12:36:48 +07:00
parent f16467eba1
commit 89c7237379
12 changed files with 361 additions and 13 deletions
+28 -1
View File
@@ -1,10 +1,11 @@
import os, uuid, json, tempfile
from fastapi import APIRouter, HTTPException, Depends, UploadFile, File
from fastapi import APIRouter, HTTPException, Depends, UploadFile, File, BackgroundTasks
from pydantic import BaseModel
from typing import Optional, Any
from app.config import settings
from app.core.vst_engine import PluginManager, HAS_PEDALBOARD, HAS_PYFLUIDSYNTH
from app.core.render_engine import PythonRenderEngine
from app.core.soundfont_inspector import SoundFontInspector
from app.api.v1.auth import get_current_user
router = APIRouter()
@@ -12,6 +13,16 @@ router = APIRouter()
UPLOAD_SF_DIR = os.path.join(settings.STORAGE_DIR, "soundfonts")
os.makedirs(UPLOAD_SF_DIR, exist_ok=True)
SYSTEM_SF_DIR = "/opt/daw_engine/soundfonts"
_inspector = None
def get_inspector():
global _inspector
if _inspector is None:
_inspector = SoundFontInspector(system_sf_dir=SYSTEM_SF_DIR, upload_sf_dir=UPLOAD_SF_DIR)
return _inspector
@router.get("/available")
async def list_plugins(current_user: dict = Depends(get_current_user)):
@@ -35,6 +46,14 @@ async def list_default_soundfonts():
return result
@router.get("/soundfonts/catalog")
async def soundfont_catalog(current_user: dict = Depends(get_current_user)):
inspector = get_inspector()
full_catalog = inspector.get_catalog()
condensed_catalog = inspector.get_condensed_catalog_summary()
return {"full_catalog": full_catalog, "condensed_catalog": condensed_catalog}
@router.get("/soundfont-instruments/{sf_id}")
async def list_soundfont_instruments(sf_id: str, current_user: dict = Depends(get_current_user)):
pm = PluginManager(upload_sf_dir=UPLOAD_SF_DIR)
@@ -45,6 +64,7 @@ async def list_soundfont_instruments(sf_id: str, current_user: dict = Depends(ge
@router.post("/upload-soundfont")
async def upload_soundfont(
file: UploadFile = File(...),
background_tasks: BackgroundTasks = None,
current_user: dict = Depends(get_current_user)
):
if not (file.filename and (file.filename.endswith(".sf2") or file.filename.endswith(".sf3"))):
@@ -69,6 +89,11 @@ async def upload_soundfont(
import json
json.dump({"original_name": file.filename, "uuid": file_uuid, "file": file_id}, f)
inspector = get_inspector()
inspector.invalidate_catalog_cache()
if background_tasks:
background_tasks.add_task(inspector.generate_full_catalog)
return {"id": file_id, "name": file.filename, "path": dest_path, "size_bytes": len(contents)}
@@ -93,6 +118,8 @@ async def delete_soundfont(sf_id: str, current_user: dict = Depends(get_current_
break
if not deleted:
raise HTTPException(status_code=404, detail="SoundFont not found")
inspector = get_inspector()
inspector.invalidate_catalog_cache()
return {"deleted": True, "sf_id": sf_id}
+47 -5
View File
@@ -6,6 +6,7 @@ from app.config import settings
from app.core.vst_engine import (
render_midi_events_to_audio,
PluginManager,
DecentSamplerManager,
HAS_PEDALBOARD,
HAS_PYFLUIDSYNTH,
)
@@ -51,10 +52,19 @@ class PythonRenderEngine:
def render_session_container(self, session: dict, section_store: dict, bpm: float, time_sig_num: int, total_samples: int) -> np.ndarray:
session_buffer = np.zeros((2, total_samples), dtype=np.float32)
_channel_counter = 0
for track in session.get("tracks", []):
track_type = track.get("type", "AUDIO")
track_buffer = np.zeros((2, total_samples), dtype=np.float32)
soundfont_bank = track.get("soundfont_bank", 0)
soundfont_program = track.get("soundfont_program", 0)
is_percussion = track.get("is_percussion", False) or (soundfont_bank == 128)
midi_channel = 9 if is_percussion else (_channel_counter % 9)
if not is_percussion:
_channel_counter += 1
for item in track.get("items", []):
start_sample = self.bars_to_samples(item["start_bar"], bpm, time_sig_num)
dur_samples = self.bars_to_samples(item["duration_bars"], bpm, time_sig_num)
@@ -124,11 +134,43 @@ class PythonRenderEngine:
plugin_mgr = PluginManager()
vst = plugin_mgr.load_vst(instrument_id) if instrument_id else None
if vst and HAS_PEDALBOARD:
instrument_source = track.get("instrument_source", "soundfont")
if instrument_source == "pianobook":
dspreset_path = track.get("dspreset_path", "")
if dspreset_path and os.path.exists(dspreset_path) and HAS_PEDALBOARD:
from pedalboard import Pedalboard
ds_manager = DecentSamplerManager()
try:
vst = ds_manager.create_decent_sampler_instance(dspreset_path)
midi_messages = PluginManager.midi_events_to_messages(
midi_events, bpm, self.sample_rate
)
total_needed = 0
for ev in midi_events:
end_sec = (ev.get("start_beat", 0) + ev.get("duration_beats", 1)) * (60.0 / bpm)
dur_samples = int(end_sec * self.sample_rate)
if dur_samples > total_needed:
total_needed = dur_samples
total_needed = max(total_needed, 1024)
silent = np.zeros((2, total_needed), dtype=np.float32)
board = Pedalboard([vst])
synth_buffer = board(silent, sample_rate=self.sample_rate, midi_messages=midi_messages)
except Exception as e:
print(f"[RenderEngine] DecentSampler/Pianobook error: {e}")
synth_buffer = render_midi_events_to_audio(
midi_events=midi_events, sr=self.sample_rate, bpm=bpm, instrument='synth'
)
else:
synth_buffer = render_midi_events_to_audio(
midi_events=midi_events, sr=self.sample_rate, bpm=bpm, instrument='synth'
)
elif vst and HAS_PEDALBOARD:
from pedalboard import Pedalboard
# Convert MIDI events with precise sample offset
midi_messages = PluginManager.midi_events_to_messages(
midi_events, bpm, self.sample_rate
midi_events, bpm, self.sample_rate,
bank=soundfont_bank, program=soundfont_program
)
total_needed = 0
for ev in midi_events:
@@ -149,7 +191,7 @@ class PythonRenderEngine:
import fluidsynth
fl = fluidsynth.FluidSynth(sample_rate=self.sample_rate, gain=0.5)
fid = fl.sfload(sf_path)
fl.program_select(0, fid, 0, 0)
fl.program_select(midi_channel, fid, soundfont_bank, soundfont_program)
beat_sec = 60.0 / bpm
total_sec = 0
for ev in midi_events:
@@ -165,11 +207,11 @@ class PythonRenderEngine:
dur_beats = ev.get("duration_beats", 1.0)
start_sec = start_beat * beat_sec
dur_sec = dur_beats * beat_sec
fl.noteon(0, note, velocity)
fl.noteon(midi_channel, note, velocity)
start_s = int(start_sec * self.sample_rate)
dur_s = int(dur_sec * self.sample_rate)
block = fl.get_samples(int(dur_s)) if hasattr(fl, 'get_samples') else np.zeros((2, dur_s), dtype=np.float32)
fl.noteoff(0, note)
fl.noteoff(midi_channel, note)
if block.shape[1] > 0:
end_s = min(start_s + block.shape[1], total_samples)
actual = end_s - start_s
+155
View File
@@ -0,0 +1,155 @@
import os
import json
import logging
logger = logging.getLogger(__name__)
try:
from sf2utils.sf2parse import Sf2File
HAS_SF2UTILS = True
except ImportError:
HAS_SF2UTILS = False
GM_CATEGORIES = [
("Piano", range(0, 8)),
("Chromatic Percussion", range(8, 16)),
("Organ", range(16, 24)),
("Guitar", range(24, 32)),
("Bass", range(32, 40)),
("Strings", range(40, 48)),
("Ensemble", range(48, 56)),
("Brass", range(56, 64)),
("Reed", range(64, 72)),
("Pipe", range(72, 80)),
("Synth Lead", range(80, 90)),
("Synth Pad", range(90, 104)),
]
MAX_CONDENSED_ENTRIES = 50
class SoundFontInspector:
def __init__(self, system_sf_dir="/opt/daw_engine/soundfonts", upload_sf_dir=None):
self.system_sf_dir = system_sf_dir
self.upload_sf_dir = upload_sf_dir
self._catalog_cache = None
def invalidate_catalog_cache(self):
self._catalog_cache = None
def inspect_sf2_file(self, filepath: str) -> dict:
if not HAS_SF2UTILS:
logger.warning("sf2utils not installed, cannot inspect .sf2 files")
return {}
if not os.path.exists(filepath):
return {}
try:
sf_name = os.path.basename(filepath)
sf_id = os.path.splitext(sf_name)[0].lower()
instruments = []
with open(filepath, 'rb') as f:
sf2 = Sf2File(f)
for preset in sf2.presets:
name = preset.name.strip()
if name == "EOP" or (preset.bank == 128 and preset.preset == 127):
continue
instruments.append({
"bank": preset.bank,
"program": preset.preset,
"name": name,
"is_percussion": (preset.bank == 128)
})
return {
"soundfont_id": sf_id,
"filename": sf_name,
"total_instruments": len(instruments),
"instruments": instruments
}
except Exception as e:
logger.warning(f"Skipping corrupted .sf2 file {filepath}: {e}")
return {}
def _scan_directory(self, directory: str) -> dict:
catalog = {}
if not os.path.isdir(directory):
return catalog
for fname in os.listdir(directory):
if not fname.lower().endswith(('.sf2', '.sf3')):
continue
full_path = os.path.join(directory, fname)
sf_info = self.inspect_sf2_file(full_path)
if sf_info and sf_info.get("soundfont_id"):
catalog[sf_info["soundfont_id"]] = sf_info
return catalog
def generate_full_catalog(self, output_json_path: str = None) -> dict:
catalog = {}
catalog.update(self._scan_directory(self.system_sf_dir))
if self.upload_sf_dir and os.path.isdir(self.upload_sf_dir):
catalog.update(self._scan_directory(self.upload_sf_dir))
if output_json_path:
os.makedirs(os.path.dirname(output_json_path), exist_ok=True)
with open(output_json_path, 'w', encoding='utf-8') as f:
json.dump(catalog, f, ensure_ascii=False, indent=2)
self._catalog_cache = catalog
return catalog
def get_catalog(self) -> dict:
if self._catalog_cache is not None:
return self._catalog_cache
return self.generate_full_catalog()
def get_condensed_catalog_summary(self) -> dict:
catalog = self.get_catalog()
condensed = {}
for sf_id, sf_info in catalog.items():
instruments = sf_info.get("instruments", [])
if not instruments:
continue
selected = []
used_programs = set()
for cat_name, prog_range in GM_CATEGORIES:
cat_members = [
inst for inst in instruments
if inst["program"] in prog_range and not inst["is_percussion"]
]
if cat_members:
representative = cat_members[0]
key = (representative["program"], representative["bank"])
if key not in used_programs:
used_programs.add(key)
selected.append(representative)
drum_kits = [inst for inst in instruments if inst["is_percussion"]]
for dk in drum_kits[:3]:
key = (dk["program"], dk["bank"])
if key not in used_programs:
used_programs.add(key)
selected.append(dk)
if len(selected) > MAX_CONDENSED_ENTRIES:
selected = selected[:MAX_CONDENSED_ENTRIES]
condensed[sf_id] = {
"soundfont_id": sf_info["soundfont_id"],
"filename": sf_info["filename"],
"total_instruments": sf_info["total_instruments"],
"condensed_count": len(selected),
"instruments": selected
}
return condensed
def format_condensed_for_prompt(self) -> str:
condensed = self.get_condensed_catalog_summary()
lines = []
for sf_id, info in condensed.items():
lines.append(f"SoundFont ID: '{sf_id}' (File: {info['filename']}):")
for inst in info["instruments"]:
lines.append(f" - {inst['name']}: bank={inst['bank']}, program={inst['program']}")
return "\n".join(lines)
+31 -1
View File
@@ -286,11 +286,15 @@ class PluginManager:
}
@staticmethod
def midi_events_to_messages(midi_events: list, bpm: float, sr: int) -> list:
def midi_events_to_messages(midi_events: list, bpm: float, sr: int, bank: int = None, program: int = None) -> list:
if not HAS_PEDALBOARD:
return []
beat_duration_sec = 60.0 / max(30.0, bpm)
messages = []
if bank is not None:
messages.append(MidiMessage(control_change=0, value=bank, sample_offset=0))
if program is not None:
messages.append(MidiMessage(program_change=program, sample_offset=0))
for ev in midi_events:
note = ev.get("note", 60)
velocity = ev.get("velocity", 100)
@@ -313,3 +317,29 @@ class PluginManager:
if data[8:12] != b'sfbk':
return False
return True
class DecentSamplerManager:
def __init__(self, vst_path="/opt/daw_engine/vst3/DecentSampler.vst3"):
self.vst_path = vst_path
def create_decent_sampler_instance(self, dspreset_path: str):
if not HAS_PEDALBOARD:
raise RuntimeError("pedalboard not available")
if not os.path.exists(self.vst_path):
raise FileNotFoundError(f"DecentSampler VST3 not found at {self.vst_path}")
if not os.path.exists(dspreset_path):
raise FileNotFoundError(f"Preset file not found at {dspreset_path}")
plugin = VST3Plugin(self.vst_path)
abs_preset = os.path.abspath(dspreset_path)
preset_dir = os.path.dirname(abs_preset)
cwd_before = os.getcwd()
try:
os.chdir(preset_dir)
plugin.load_preset(abs_preset)
finally:
os.chdir(cwd_before)
return plugin
+17 -1
View File
@@ -3351,9 +3351,13 @@ const PluginManagerModal = ({ isOpen, onClose, pluginsData }) => {
try {
const result = await window.SonicAPI.uploadSoundFont(file);
setSfUploadStatus('Uploaded: ' + result.name);
// Refresh plugin list
// Refresh plugin list and catalog
const data = await window.SonicAPI.listPlugins();
setLocalData(data);
try {
const cat = await window.SonicAPI.getSoundfontCatalog();
window.__soundfontCatalog = cat;
} catch (_) {}
} catch (err) {
setSfUploadStatus('Error: ' + err.message);
}
@@ -7027,6 +7031,11 @@ const App = () => {
if (active) setSelectedProviderId(active.id);
}
} catch (e) { }
try {
window.SonicAPI.getSoundfontCatalog().then(cat => {
window.__soundfontCatalog = cat;
}).catch(() => {});
} catch (e) { }
})();
}
};
@@ -13802,6 +13811,13 @@ const App = () => {
}))
};
targetTrack.midiItems = [...(targetTrack.midiItems || []), newMidiItem];
if (aiTrack.soundfont_bank !== undefined && aiTrack.soundfont_program !== undefined) {
targetTrack.soundfont_bank = aiTrack.soundfont_bank;
targetTrack.soundfont_program = aiTrack.soundfont_program;
if (window.SonicSF && window.SonicSF.applyAITrackInstrument) {
window.SonicSF.applyAITrackInstrument(aiTrack.soundfont_bank, aiTrack.soundfont_program);
}
}
});
return updatedTracks;
});
+22 -3
View File
@@ -48,7 +48,7 @@ const AIGateway = (function() {
name: 'fade_out', description: 'Fade-out clip (0.5s đến max)', parameters: { type: 'object', properties: { track_id: { type: 'string' }, duration_seconds: { type: 'number' }, clip_index: { type: 'number', description: 'Chỉ số của clip trên track (1-based, ví dụ: 1 cho clip 1, 2 cho clip 2)' }, clip_id: { type: 'string', description: 'ID của clip cụ thể' } } }
}, {
name: 'generate_multitrack_midi',
description: 'Generates multi-track MIDI data based on genre, bar duration, and requested instruments list.',
description: 'Generates multi-track MIDI data along with SoundFont Program configurations for each track.',
parameters: {
type: 'object',
properties: {
@@ -57,12 +57,15 @@ const AIGateway = (function() {
total_bars: { type: 'integer' },
tracks: {
type: 'array',
description: 'Array of instrument tracks along with their corresponding MIDI notes',
description: 'Array of instrument tracks with MIDI notes and SoundFont instrument selection',
items: {
type: 'object',
properties: {
track_name: { type: 'string', description: 'Track name (e.g., String Ensemble, Epic Brass, Taiko Drums)' },
instrument_type: { type: 'string', enum: ['STRINGS', 'BRASS', 'SYNTH', 'PERCUSSION', 'DRUMS'] },
soundfont_id: { type: 'string', description: "ID of the SoundFont to use (e.g. 'generaluser_gs')" },
soundfont_bank: { type: 'integer', default: 0, description: 'MIDI Bank code: 0 for melodic instruments, 128 for Drums/Percussion' },
soundfont_program: { type: 'integer', description: 'MIDI Program Number 0-127 matching the instrument name in the SoundFont catalog' },
notes: {
type: 'array',
items: {
@@ -77,7 +80,7 @@ const AIGateway = (function() {
}
}
},
required: ['track_name', 'instrument_type', 'notes']
required: ['track_name', 'instrument_type', 'soundfont_id', 'soundfont_bank', 'soundfont_program', 'notes']
}
}
},
@@ -176,13 +179,29 @@ const AIGateway = (function() {
return calls;
}
function buildCatalogPromptSection() {
const catalog = window.__soundfontCatalog;
if (!catalog || !catalog.condensed_catalog) return '';
const lines = [];
for (const [sfId, info] of Object.entries(catalog.condensed_catalog)) {
lines.push(`SoundFont ID: '${sfId}' (File: ${info.filename}):`);
for (const inst of info.instruments || []) {
lines.push(` - ${inst.name}: bank=${inst.bank}, program=${inst.program}`);
}
}
if (lines.length === 0) return '';
return `\n=== SOUNDFONT INSTRUMENT CATALOG ===\nYou have the following SoundFont instruments available on the system:\n${lines.join('\n')}\n\nMANDATORY RULES WHEN CREATING TRACKS WITH generate_multitrack_midi:\n1. You MUST look up the catalog above and fill in the correct soundfont_id, soundfont_bank, and soundfont_program for each track.\n2. Melodic instruments (Piano, Strings, Brass, etc.) MUST use soundfont_bank=0.\n3. Drums and Percussion MUST use soundfont_bank=128.\n4. Example: For \"Brass horns\", use soundfont_id="generaluser_gs", soundfont_bank=0, soundfont_program=56.\n5. Example: For \"Drum kit\", use soundfont_id="generaluser_gs", soundfont_bank=128, soundfont_program=0.\n`;
}
function buildUserMessage(prompt, context, systemInstruction = '') {
const contextStr = JSON.stringify(context, null, 2);
const toolNames = DEFAULT_TOOLS.map(t => ` - ${t.name}: ${t.description}`).join('\n');
const catalogSection = buildCatalogPromptSection();
return [
{ role: 'system', content: `Bạn là trợ lý điều khiển DAW chuyên nghiệp.
Nhiệm vụ của bạn là phân tích yêu cầu của người dùng và chuyển đổi thành danh sách các function calls tương ứng.
${systemInstruction ? `\nHướng dẫn tạo nhạc đặc biệt từ Preset:\n${systemInstruction}\n` : ''}
${catalogSection}
QUAN TRỌNG:
1. Bạn đang hoạt động ở chế độ một lượt (one-shot). Hãy trả về TẤT CẢ các function calls cần thiết để thực hiện toàn bộ các bước trong yêu cầu của người dùng trong một phản hồi duy nhất. Đừng thực hiện từng bước qua nhiều lượt chat.
2. Có thể gọi nhiều function cùng một lúc (gọi song song/nối tiếp). Chúng sẽ được thực thi theo thứ tự bạn trả về.
+1
View File
@@ -63,6 +63,7 @@ window.API_BASE_URL = window.API_BASE_URL || window.location.origin;
savePreferences: (prefs) => apiRequest('/api/v1/user/preferences', { method: 'POST', body: JSON.stringify({ preferences: prefs }) }),
listPlugins: () => apiRequest('/api/v1/plugins/available', { method: 'GET' }),
getSoundfontCatalog: () => apiRequest('/api/v1/plugins/soundfonts/catalog', { method: 'GET' }),
listDefaultSoundfonts: () => apiRequest('/api/v1/plugins/default-soundfonts', { method: 'GET' }),
listSoundfontInstruments: (sfId) => apiRequest(`/api/v1/plugins/soundfont-instruments/${sfId}`, { method: 'GET' }),
renderProject: (projectJson, outputFilename) => apiRequest('/api/v1/plugins/render', { method: 'POST', body: JSON.stringify({ project_json: projectJson, output_filename: outputFilename }) }),
+42 -2
View File
@@ -30,9 +30,45 @@
return window.__sharedAudioCtx;
};
// ── Per-channel MIDI state (16 GM channels) ──
const _channels = Array.from({ length: 16 }, () => ({ bank: 0, program: 0, isPercussion: false }));
let _nextMelodicChannel = 0;
const SonicSF = {
loadedFonts: {},
controllerChange: function (channel, controller, value) {
if (channel < 0 || channel > 15) return;
if (controller === 0) {
_channels[channel].bank = value;
_channels[channel].isPercussion = (value === 128);
}
},
programChange: function (channel, program) {
if (channel < 0 || channel > 15) return;
_channels[channel].program = program;
},
allocateChannel: function (bank) {
if (bank === 128) return 9;
const ch = _nextMelodicChannel % 9;
_nextMelodicChannel = (_nextMelodicChannel + 1) % 9;
return ch;
},
applyAITrackInstrument: function (bank, program) {
const channel = this.allocateChannel(bank);
this.controllerChange(channel, 0, bank);
this.programChange(channel, program);
return channel;
},
getChannelState: function (channel) {
if (channel < 0 || channel > 15) return null;
return { ..._channels[channel] };
},
// Load SoundFont from URL → ArrayBuffer → store in memory
loadSoundFont: async function (url) {
if (this.loadedFonts[url]) return this.loadedFonts[url];
@@ -43,7 +79,7 @@
return buffer;
},
playNote: function (note, velocity, durationMs, startTime, program, destinationNode) {
playNote: function (note, velocity, durationMs, startTime, program, destinationNode, channel) {
const ctx = getCtx();
const freq = 440 * Math.pow(2, (note - 69) / 12);
if (freq <= 0 || isNaN(freq)) return null;
@@ -59,7 +95,11 @@
let releaseTime = 0.2;
let volFactor = 0.25;
const prog = program !== undefined ? parseInt(program) : 0;
let prog = program !== undefined ? parseInt(program) : 0;
if (channel !== undefined && channel >= 0 && channel < 16) {
const chState = _channels[channel];
prog = chState.program || prog;
}
if (prog >= 0 && prog <= 7) { // Pianos
oscType = 'sine';
decayTime = 0.3;