feat: add SoundFont inspection engine + AI instrument schema
- SoundFontInspector (sf2utils) scans .sf2, generates full/condensed catalog - GET /api/v1/plugins/soundfonts/catalog with lazy init + cache invalidation - AI tool generate_multitrack_midi now requires soundfont_id/bank/program - Condensed catalog auto-injected into AI system prompt with bank rules - Server render: FluidSynth program_select uses bank/program + channel routing (drums→ch9) - VST3 pedalboard path inserts CC0 bank select + program change before notes - DecentSamplerManager loads .dspreset with CWD fix for relative sample paths - Pianobook render branch in render_engine.py - Client SonicSF: controllerChange, programChange, applyAITrackInstrument - Post-AI track creation applies instrument via applyAITrackInstrument - Background cache rescan on .sf2 upload, frontend re-fetches catalog - libcurl4 + VST3 dirs in Dockerfile
This commit is contained in:
@@ -6,6 +6,7 @@ from app.config import settings
|
||||
from app.core.vst_engine import (
|
||||
render_midi_events_to_audio,
|
||||
PluginManager,
|
||||
DecentSamplerManager,
|
||||
HAS_PEDALBOARD,
|
||||
HAS_PYFLUIDSYNTH,
|
||||
)
|
||||
@@ -51,10 +52,19 @@ class PythonRenderEngine:
|
||||
def render_session_container(self, session: dict, section_store: dict, bpm: float, time_sig_num: int, total_samples: int) -> np.ndarray:
|
||||
session_buffer = np.zeros((2, total_samples), dtype=np.float32)
|
||||
|
||||
_channel_counter = 0
|
||||
|
||||
for track in session.get("tracks", []):
|
||||
track_type = track.get("type", "AUDIO")
|
||||
track_buffer = np.zeros((2, total_samples), dtype=np.float32)
|
||||
|
||||
soundfont_bank = track.get("soundfont_bank", 0)
|
||||
soundfont_program = track.get("soundfont_program", 0)
|
||||
is_percussion = track.get("is_percussion", False) or (soundfont_bank == 128)
|
||||
midi_channel = 9 if is_percussion else (_channel_counter % 9)
|
||||
if not is_percussion:
|
||||
_channel_counter += 1
|
||||
|
||||
for item in track.get("items", []):
|
||||
start_sample = self.bars_to_samples(item["start_bar"], bpm, time_sig_num)
|
||||
dur_samples = self.bars_to_samples(item["duration_bars"], bpm, time_sig_num)
|
||||
@@ -124,11 +134,43 @@ class PythonRenderEngine:
|
||||
plugin_mgr = PluginManager()
|
||||
vst = plugin_mgr.load_vst(instrument_id) if instrument_id else None
|
||||
|
||||
if vst and HAS_PEDALBOARD:
|
||||
instrument_source = track.get("instrument_source", "soundfont")
|
||||
|
||||
if instrument_source == "pianobook":
|
||||
dspreset_path = track.get("dspreset_path", "")
|
||||
if dspreset_path and os.path.exists(dspreset_path) and HAS_PEDALBOARD:
|
||||
from pedalboard import Pedalboard
|
||||
ds_manager = DecentSamplerManager()
|
||||
try:
|
||||
vst = ds_manager.create_decent_sampler_instance(dspreset_path)
|
||||
midi_messages = PluginManager.midi_events_to_messages(
|
||||
midi_events, bpm, self.sample_rate
|
||||
)
|
||||
total_needed = 0
|
||||
for ev in midi_events:
|
||||
end_sec = (ev.get("start_beat", 0) + ev.get("duration_beats", 1)) * (60.0 / bpm)
|
||||
dur_samples = int(end_sec * self.sample_rate)
|
||||
if dur_samples > total_needed:
|
||||
total_needed = dur_samples
|
||||
total_needed = max(total_needed, 1024)
|
||||
silent = np.zeros((2, total_needed), dtype=np.float32)
|
||||
board = Pedalboard([vst])
|
||||
synth_buffer = board(silent, sample_rate=self.sample_rate, midi_messages=midi_messages)
|
||||
except Exception as e:
|
||||
print(f"[RenderEngine] DecentSampler/Pianobook error: {e}")
|
||||
synth_buffer = render_midi_events_to_audio(
|
||||
midi_events=midi_events, sr=self.sample_rate, bpm=bpm, instrument='synth'
|
||||
)
|
||||
else:
|
||||
synth_buffer = render_midi_events_to_audio(
|
||||
midi_events=midi_events, sr=self.sample_rate, bpm=bpm, instrument='synth'
|
||||
)
|
||||
elif vst and HAS_PEDALBOARD:
|
||||
from pedalboard import Pedalboard
|
||||
# Convert MIDI events with precise sample offset
|
||||
midi_messages = PluginManager.midi_events_to_messages(
|
||||
midi_events, bpm, self.sample_rate
|
||||
midi_events, bpm, self.sample_rate,
|
||||
bank=soundfont_bank, program=soundfont_program
|
||||
)
|
||||
total_needed = 0
|
||||
for ev in midi_events:
|
||||
@@ -149,7 +191,7 @@ class PythonRenderEngine:
|
||||
import fluidsynth
|
||||
fl = fluidsynth.FluidSynth(sample_rate=self.sample_rate, gain=0.5)
|
||||
fid = fl.sfload(sf_path)
|
||||
fl.program_select(0, fid, 0, 0)
|
||||
fl.program_select(midi_channel, fid, soundfont_bank, soundfont_program)
|
||||
beat_sec = 60.0 / bpm
|
||||
total_sec = 0
|
||||
for ev in midi_events:
|
||||
@@ -165,11 +207,11 @@ class PythonRenderEngine:
|
||||
dur_beats = ev.get("duration_beats", 1.0)
|
||||
start_sec = start_beat * beat_sec
|
||||
dur_sec = dur_beats * beat_sec
|
||||
fl.noteon(0, note, velocity)
|
||||
fl.noteon(midi_channel, note, velocity)
|
||||
start_s = int(start_sec * self.sample_rate)
|
||||
dur_s = int(dur_sec * self.sample_rate)
|
||||
block = fl.get_samples(int(dur_s)) if hasattr(fl, 'get_samples') else np.zeros((2, dur_s), dtype=np.float32)
|
||||
fl.noteoff(0, note)
|
||||
fl.noteoff(midi_channel, note)
|
||||
if block.shape[1] > 0:
|
||||
end_s = min(start_s + block.shape[1], total_samples)
|
||||
actual = end_s - start_s
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
from sf2utils.sf2parse import Sf2File
|
||||
HAS_SF2UTILS = True
|
||||
except ImportError:
|
||||
HAS_SF2UTILS = False
|
||||
|
||||
GM_CATEGORIES = [
|
||||
("Piano", range(0, 8)),
|
||||
("Chromatic Percussion", range(8, 16)),
|
||||
("Organ", range(16, 24)),
|
||||
("Guitar", range(24, 32)),
|
||||
("Bass", range(32, 40)),
|
||||
("Strings", range(40, 48)),
|
||||
("Ensemble", range(48, 56)),
|
||||
("Brass", range(56, 64)),
|
||||
("Reed", range(64, 72)),
|
||||
("Pipe", range(72, 80)),
|
||||
("Synth Lead", range(80, 90)),
|
||||
("Synth Pad", range(90, 104)),
|
||||
]
|
||||
|
||||
MAX_CONDENSED_ENTRIES = 50
|
||||
|
||||
|
||||
class SoundFontInspector:
|
||||
def __init__(self, system_sf_dir="/opt/daw_engine/soundfonts", upload_sf_dir=None):
|
||||
self.system_sf_dir = system_sf_dir
|
||||
self.upload_sf_dir = upload_sf_dir
|
||||
self._catalog_cache = None
|
||||
|
||||
def invalidate_catalog_cache(self):
|
||||
self._catalog_cache = None
|
||||
|
||||
def inspect_sf2_file(self, filepath: str) -> dict:
|
||||
if not HAS_SF2UTILS:
|
||||
logger.warning("sf2utils not installed, cannot inspect .sf2 files")
|
||||
return {}
|
||||
if not os.path.exists(filepath):
|
||||
return {}
|
||||
|
||||
try:
|
||||
sf_name = os.path.basename(filepath)
|
||||
sf_id = os.path.splitext(sf_name)[0].lower()
|
||||
|
||||
instruments = []
|
||||
with open(filepath, 'rb') as f:
|
||||
sf2 = Sf2File(f)
|
||||
for preset in sf2.presets:
|
||||
name = preset.name.strip()
|
||||
if name == "EOP" or (preset.bank == 128 and preset.preset == 127):
|
||||
continue
|
||||
instruments.append({
|
||||
"bank": preset.bank,
|
||||
"program": preset.preset,
|
||||
"name": name,
|
||||
"is_percussion": (preset.bank == 128)
|
||||
})
|
||||
|
||||
return {
|
||||
"soundfont_id": sf_id,
|
||||
"filename": sf_name,
|
||||
"total_instruments": len(instruments),
|
||||
"instruments": instruments
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"Skipping corrupted .sf2 file {filepath}: {e}")
|
||||
return {}
|
||||
|
||||
def _scan_directory(self, directory: str) -> dict:
|
||||
catalog = {}
|
||||
if not os.path.isdir(directory):
|
||||
return catalog
|
||||
for fname in os.listdir(directory):
|
||||
if not fname.lower().endswith(('.sf2', '.sf3')):
|
||||
continue
|
||||
full_path = os.path.join(directory, fname)
|
||||
sf_info = self.inspect_sf2_file(full_path)
|
||||
if sf_info and sf_info.get("soundfont_id"):
|
||||
catalog[sf_info["soundfont_id"]] = sf_info
|
||||
return catalog
|
||||
|
||||
def generate_full_catalog(self, output_json_path: str = None) -> dict:
|
||||
catalog = {}
|
||||
catalog.update(self._scan_directory(self.system_sf_dir))
|
||||
if self.upload_sf_dir and os.path.isdir(self.upload_sf_dir):
|
||||
catalog.update(self._scan_directory(self.upload_sf_dir))
|
||||
|
||||
if output_json_path:
|
||||
os.makedirs(os.path.dirname(output_json_path), exist_ok=True)
|
||||
with open(output_json_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(catalog, f, ensure_ascii=False, indent=2)
|
||||
|
||||
self._catalog_cache = catalog
|
||||
return catalog
|
||||
|
||||
def get_catalog(self) -> dict:
|
||||
if self._catalog_cache is not None:
|
||||
return self._catalog_cache
|
||||
return self.generate_full_catalog()
|
||||
|
||||
def get_condensed_catalog_summary(self) -> dict:
|
||||
catalog = self.get_catalog()
|
||||
condensed = {}
|
||||
for sf_id, sf_info in catalog.items():
|
||||
instruments = sf_info.get("instruments", [])
|
||||
if not instruments:
|
||||
continue
|
||||
|
||||
selected = []
|
||||
used_programs = set()
|
||||
for cat_name, prog_range in GM_CATEGORIES:
|
||||
cat_members = [
|
||||
inst for inst in instruments
|
||||
if inst["program"] in prog_range and not inst["is_percussion"]
|
||||
]
|
||||
if cat_members:
|
||||
representative = cat_members[0]
|
||||
key = (representative["program"], representative["bank"])
|
||||
if key not in used_programs:
|
||||
used_programs.add(key)
|
||||
selected.append(representative)
|
||||
|
||||
drum_kits = [inst for inst in instruments if inst["is_percussion"]]
|
||||
for dk in drum_kits[:3]:
|
||||
key = (dk["program"], dk["bank"])
|
||||
if key not in used_programs:
|
||||
used_programs.add(key)
|
||||
selected.append(dk)
|
||||
|
||||
if len(selected) > MAX_CONDENSED_ENTRIES:
|
||||
selected = selected[:MAX_CONDENSED_ENTRIES]
|
||||
|
||||
condensed[sf_id] = {
|
||||
"soundfont_id": sf_info["soundfont_id"],
|
||||
"filename": sf_info["filename"],
|
||||
"total_instruments": sf_info["total_instruments"],
|
||||
"condensed_count": len(selected),
|
||||
"instruments": selected
|
||||
}
|
||||
return condensed
|
||||
|
||||
def format_condensed_for_prompt(self) -> str:
|
||||
condensed = self.get_condensed_catalog_summary()
|
||||
lines = []
|
||||
for sf_id, info in condensed.items():
|
||||
lines.append(f"SoundFont ID: '{sf_id}' (File: {info['filename']}):")
|
||||
for inst in info["instruments"]:
|
||||
lines.append(f" - {inst['name']}: bank={inst['bank']}, program={inst['program']}")
|
||||
return "\n".join(lines)
|
||||
+31
-1
@@ -286,11 +286,15 @@ class PluginManager:
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def midi_events_to_messages(midi_events: list, bpm: float, sr: int) -> list:
|
||||
def midi_events_to_messages(midi_events: list, bpm: float, sr: int, bank: int = None, program: int = None) -> list:
|
||||
if not HAS_PEDALBOARD:
|
||||
return []
|
||||
beat_duration_sec = 60.0 / max(30.0, bpm)
|
||||
messages = []
|
||||
if bank is not None:
|
||||
messages.append(MidiMessage(control_change=0, value=bank, sample_offset=0))
|
||||
if program is not None:
|
||||
messages.append(MidiMessage(program_change=program, sample_offset=0))
|
||||
for ev in midi_events:
|
||||
note = ev.get("note", 60)
|
||||
velocity = ev.get("velocity", 100)
|
||||
@@ -313,3 +317,29 @@ class PluginManager:
|
||||
if data[8:12] != b'sfbk':
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class DecentSamplerManager:
|
||||
def __init__(self, vst_path="/opt/daw_engine/vst3/DecentSampler.vst3"):
|
||||
self.vst_path = vst_path
|
||||
|
||||
def create_decent_sampler_instance(self, dspreset_path: str):
|
||||
if not HAS_PEDALBOARD:
|
||||
raise RuntimeError("pedalboard not available")
|
||||
if not os.path.exists(self.vst_path):
|
||||
raise FileNotFoundError(f"DecentSampler VST3 not found at {self.vst_path}")
|
||||
if not os.path.exists(dspreset_path):
|
||||
raise FileNotFoundError(f"Preset file not found at {dspreset_path}")
|
||||
|
||||
plugin = VST3Plugin(self.vst_path)
|
||||
|
||||
abs_preset = os.path.abspath(dspreset_path)
|
||||
preset_dir = os.path.dirname(abs_preset)
|
||||
cwd_before = os.getcwd()
|
||||
try:
|
||||
os.chdir(preset_dir)
|
||||
plugin.load_preset(abs_preset)
|
||||
finally:
|
||||
os.chdir(cwd_before)
|
||||
|
||||
return plugin
|
||||
|
||||
Reference in New Issue
Block a user