feat: add SoundFont inspection engine + AI instrument schema
- SoundFontInspector (sf2utils) scans .sf2, generates full/condensed catalog - GET /api/v1/plugins/soundfonts/catalog with lazy init + cache invalidation - AI tool generate_multitrack_midi now requires soundfont_id/bank/program - Condensed catalog auto-injected into AI system prompt with bank rules - Server render: FluidSynth program_select uses bank/program + channel routing (drums→ch9) - VST3 pedalboard path inserts CC0 bank select + program change before notes - DecentSamplerManager loads .dspreset with CWD fix for relative sample paths - Pianobook render branch in render_engine.py - Client SonicSF: controllerChange, programChange, applyAITrackInstrument - Post-AI track creation applies instrument via applyAITrackInstrument - Background cache rescan on .sf2 upload, frontend re-fetches catalog - libcurl4 + VST3 dirs in Dockerfile
This commit is contained in:
@@ -6,6 +6,7 @@ from app.config import settings
|
||||
from app.core.vst_engine import (
|
||||
render_midi_events_to_audio,
|
||||
PluginManager,
|
||||
DecentSamplerManager,
|
||||
HAS_PEDALBOARD,
|
||||
HAS_PYFLUIDSYNTH,
|
||||
)
|
||||
@@ -51,10 +52,19 @@ class PythonRenderEngine:
|
||||
def render_session_container(self, session: dict, section_store: dict, bpm: float, time_sig_num: int, total_samples: int) -> np.ndarray:
|
||||
session_buffer = np.zeros((2, total_samples), dtype=np.float32)
|
||||
|
||||
_channel_counter = 0
|
||||
|
||||
for track in session.get("tracks", []):
|
||||
track_type = track.get("type", "AUDIO")
|
||||
track_buffer = np.zeros((2, total_samples), dtype=np.float32)
|
||||
|
||||
soundfont_bank = track.get("soundfont_bank", 0)
|
||||
soundfont_program = track.get("soundfont_program", 0)
|
||||
is_percussion = track.get("is_percussion", False) or (soundfont_bank == 128)
|
||||
midi_channel = 9 if is_percussion else (_channel_counter % 9)
|
||||
if not is_percussion:
|
||||
_channel_counter += 1
|
||||
|
||||
for item in track.get("items", []):
|
||||
start_sample = self.bars_to_samples(item["start_bar"], bpm, time_sig_num)
|
||||
dur_samples = self.bars_to_samples(item["duration_bars"], bpm, time_sig_num)
|
||||
@@ -124,11 +134,43 @@ class PythonRenderEngine:
|
||||
plugin_mgr = PluginManager()
|
||||
vst = plugin_mgr.load_vst(instrument_id) if instrument_id else None
|
||||
|
||||
if vst and HAS_PEDALBOARD:
|
||||
instrument_source = track.get("instrument_source", "soundfont")
|
||||
|
||||
if instrument_source == "pianobook":
|
||||
dspreset_path = track.get("dspreset_path", "")
|
||||
if dspreset_path and os.path.exists(dspreset_path) and HAS_PEDALBOARD:
|
||||
from pedalboard import Pedalboard
|
||||
ds_manager = DecentSamplerManager()
|
||||
try:
|
||||
vst = ds_manager.create_decent_sampler_instance(dspreset_path)
|
||||
midi_messages = PluginManager.midi_events_to_messages(
|
||||
midi_events, bpm, self.sample_rate
|
||||
)
|
||||
total_needed = 0
|
||||
for ev in midi_events:
|
||||
end_sec = (ev.get("start_beat", 0) + ev.get("duration_beats", 1)) * (60.0 / bpm)
|
||||
dur_samples = int(end_sec * self.sample_rate)
|
||||
if dur_samples > total_needed:
|
||||
total_needed = dur_samples
|
||||
total_needed = max(total_needed, 1024)
|
||||
silent = np.zeros((2, total_needed), dtype=np.float32)
|
||||
board = Pedalboard([vst])
|
||||
synth_buffer = board(silent, sample_rate=self.sample_rate, midi_messages=midi_messages)
|
||||
except Exception as e:
|
||||
print(f"[RenderEngine] DecentSampler/Pianobook error: {e}")
|
||||
synth_buffer = render_midi_events_to_audio(
|
||||
midi_events=midi_events, sr=self.sample_rate, bpm=bpm, instrument='synth'
|
||||
)
|
||||
else:
|
||||
synth_buffer = render_midi_events_to_audio(
|
||||
midi_events=midi_events, sr=self.sample_rate, bpm=bpm, instrument='synth'
|
||||
)
|
||||
elif vst and HAS_PEDALBOARD:
|
||||
from pedalboard import Pedalboard
|
||||
# Convert MIDI events with precise sample offset
|
||||
midi_messages = PluginManager.midi_events_to_messages(
|
||||
midi_events, bpm, self.sample_rate
|
||||
midi_events, bpm, self.sample_rate,
|
||||
bank=soundfont_bank, program=soundfont_program
|
||||
)
|
||||
total_needed = 0
|
||||
for ev in midi_events:
|
||||
@@ -149,7 +191,7 @@ class PythonRenderEngine:
|
||||
import fluidsynth
|
||||
fl = fluidsynth.FluidSynth(sample_rate=self.sample_rate, gain=0.5)
|
||||
fid = fl.sfload(sf_path)
|
||||
fl.program_select(0, fid, 0, 0)
|
||||
fl.program_select(midi_channel, fid, soundfont_bank, soundfont_program)
|
||||
beat_sec = 60.0 / bpm
|
||||
total_sec = 0
|
||||
for ev in midi_events:
|
||||
@@ -165,11 +207,11 @@ class PythonRenderEngine:
|
||||
dur_beats = ev.get("duration_beats", 1.0)
|
||||
start_sec = start_beat * beat_sec
|
||||
dur_sec = dur_beats * beat_sec
|
||||
fl.noteon(0, note, velocity)
|
||||
fl.noteon(midi_channel, note, velocity)
|
||||
start_s = int(start_sec * self.sample_rate)
|
||||
dur_s = int(dur_sec * self.sample_rate)
|
||||
block = fl.get_samples(int(dur_s)) if hasattr(fl, 'get_samples') else np.zeros((2, dur_s), dtype=np.float32)
|
||||
fl.noteoff(0, note)
|
||||
fl.noteoff(midi_channel, note)
|
||||
if block.shape[1] > 0:
|
||||
end_s = min(start_s + block.shape[1], total_samples)
|
||||
actual = end_s - start_s
|
||||
|
||||
Reference in New Issue
Block a user