feat(engine): fix SF2 path resolution, add synth_engine struct and fallback chain
- Fix SF2 path: search upload dir, system dir, static dir (backward compat) - Add synth_engine struct parsing with flat field fallback - Add 3-level fallback: selected SF -> default SF -> oscillator synth - Write synth_engine in setTrackInstrument UI setters - Pass synth_engine context to client SoundFontPlayer and AI gateway
This commit is contained in:
+60
-17
@@ -1,4 +1,4 @@
|
||||
import os
|
||||
import os, logging
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import scipy.signal as signal
|
||||
@@ -11,6 +11,43 @@ from app.core.vst_engine import (
|
||||
HAS_PYFLUIDSYNTH,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
UPLOAD_SF_DIR = os.path.join(settings.STORAGE_DIR, "soundfonts")
|
||||
SYSTEM_SF_DIR = "/opt/daw_engine/soundfonts"
|
||||
|
||||
SYS_SOUNDFONTS = [
|
||||
("GeneralUser_GS.sf2", "GeneralUser GS"),
|
||||
("SGM_v2.01.sf2", "SGM v2.01"),
|
||||
("SGM-V2.01.sf2", "SGM v2.01"),
|
||||
]
|
||||
|
||||
def _find_sf2_path(sf_id: str) -> str:
|
||||
clean_id = sf_id.replace("sf_", "") if sf_id.startswith("sf_") else sf_id
|
||||
for base_dir in [UPLOAD_SF_DIR, SYSTEM_SF_DIR]:
|
||||
if not os.path.isdir(base_dir):
|
||||
continue
|
||||
for fname in os.listdir(base_dir):
|
||||
fbase, fext = os.path.splitext(fname)
|
||||
if fext.lower() in (".sf2", ".sf3") and (fbase == clean_id or fbase == sf_id):
|
||||
return os.path.join(base_dir, fname)
|
||||
# Backward compat: static/soundfonts
|
||||
static_dir = os.path.join(settings.APP_DIR, "static", "soundfonts")
|
||||
if os.path.isdir(static_dir):
|
||||
for fname in os.listdir(static_dir):
|
||||
fbase, fext = os.path.splitext(fname)
|
||||
if fext.lower() in (".sf2", ".sf3") and (fbase == clean_id or fbase == sf_id):
|
||||
return os.path.join(static_dir, fname)
|
||||
return ""
|
||||
|
||||
def _find_default_sf2() -> str:
|
||||
for sf_name, _ in SYS_SOUNDFONTS:
|
||||
for base_dir in [SYSTEM_SF_DIR, UPLOAD_SF_DIR]:
|
||||
p = os.path.join(base_dir, sf_name)
|
||||
if os.path.exists(p):
|
||||
return p
|
||||
return ""
|
||||
|
||||
if HAS_PEDALBOARD:
|
||||
try:
|
||||
from pedalboard import Pedalboard, Gain, Chorus, Reverb
|
||||
@@ -58,8 +95,13 @@ class PythonRenderEngine:
|
||||
track_type = track.get("type", "AUDIO")
|
||||
track_buffer = np.zeros((2, total_samples), dtype=np.float32)
|
||||
|
||||
soundfont_bank = track.get("soundfont_bank", 0)
|
||||
soundfont_program = track.get("soundfont_program", 0)
|
||||
# Parse synth_engine struct (Task C) — fall back to flat fields
|
||||
se = track.get("synth_engine", {}) or {}
|
||||
instrument_id = se.get("plugin_id") or track.get("instrument_id", "") or track.get("instrument", "")
|
||||
instrument_source = se.get("type") or track.get("instrument_source", "soundfont")
|
||||
soundfont_bank = se.get("soundfont_bank") if se.get("soundfont_bank") is not None else track.get("soundfont_bank", 0)
|
||||
soundfont_program = se.get("soundfont_program") if se.get("soundfont_program") is not None else track.get("soundfont_program", 0)
|
||||
soundfont_id = se.get("soundfont_id") or track.get("soundfont_id", "")
|
||||
is_percussion = track.get("is_percussion", False) or (soundfont_bank == 128)
|
||||
midi_channel = 9 if is_percussion else (_channel_counter % 9)
|
||||
if not is_percussion:
|
||||
@@ -130,12 +172,9 @@ class PythonRenderEngine:
|
||||
|
||||
if midi_events:
|
||||
try:
|
||||
instrument_id = track.get("instrument_id", "") or track.get("instrument", "")
|
||||
plugin_mgr = PluginManager()
|
||||
vst = plugin_mgr.load_vst(instrument_id) if instrument_id else None
|
||||
|
||||
instrument_source = track.get("instrument_source", "soundfont")
|
||||
|
||||
if instrument_source == "pianobook":
|
||||
dspreset_path = track.get("dspreset_path", "")
|
||||
if dspreset_path and os.path.exists(dspreset_path) and HAS_PEDALBOARD:
|
||||
@@ -157,7 +196,7 @@ class PythonRenderEngine:
|
||||
board = Pedalboard([vst])
|
||||
synth_buffer = board(silent, sample_rate=self.sample_rate, midi_messages=midi_messages)
|
||||
except Exception as e:
|
||||
print(f"[RenderEngine] DecentSampler/Pianobook error: {e}")
|
||||
logger.warning(f"[RenderEngine] DecentSampler/Pianobook error: {e}")
|
||||
synth_buffer = render_midi_events_to_audio(
|
||||
midi_events=midi_events, sr=self.sample_rate, bpm=bpm, instrument='synth'
|
||||
)
|
||||
@@ -167,7 +206,6 @@ class PythonRenderEngine:
|
||||
)
|
||||
elif vst and HAS_PEDALBOARD:
|
||||
from pedalboard import Pedalboard
|
||||
# Convert MIDI events with precise sample offset
|
||||
midi_messages = PluginManager.midi_events_to_messages(
|
||||
midi_events, bpm, self.sample_rate,
|
||||
bank=soundfont_bank, program=soundfont_program
|
||||
@@ -182,12 +220,15 @@ class PythonRenderEngine:
|
||||
silent = np.zeros((2, total_needed), dtype=np.float32)
|
||||
board = Pedalboard([vst])
|
||||
synth_buffer = board(silent, sample_rate=self.sample_rate, midi_messages=midi_messages)
|
||||
elif instrument_id and instrument_id.startswith("sf_"):
|
||||
sf_path = os.path.join(
|
||||
os.path.dirname(os.path.dirname(__file__)), "static", "soundfonts",
|
||||
instrument_id.replace("sf_", "") + ".sf2"
|
||||
)
|
||||
if os.path.exists(sf_path) and HAS_PYFLUIDSYNTH:
|
||||
elif instrument_id and (instrument_id.startswith("sf_") or soundfont_id):
|
||||
sf_path = _find_sf2_path(soundfont_id or instrument_id)
|
||||
# 3-level fallback: selected SF → default SF → oscillator synth
|
||||
if not sf_path or not os.path.exists(sf_path) or not HAS_PYFLUIDSYNTH:
|
||||
if not sf_path or not os.path.exists(sf_path):
|
||||
logger.warning(f"[RenderEngine] SoundFont not found for {soundfont_id or instrument_id}, trying default")
|
||||
sf_path = _find_default_sf2() if HAS_PYFLUIDSYNTH else ""
|
||||
|
||||
if sf_path and os.path.exists(sf_path) and HAS_PYFLUIDSYNTH:
|
||||
import fluidsynth
|
||||
fl = fluidsynth.FluidSynth(sample_rate=self.sample_rate, gain=0.5)
|
||||
fid = fl.sfload(sf_path)
|
||||
@@ -198,8 +239,8 @@ class PythonRenderEngine:
|
||||
end_sec = (ev.get("start_beat", 0) + ev.get("duration_beats", 1)) * beat_sec
|
||||
if end_sec > total_sec:
|
||||
total_sec = end_sec
|
||||
total_samples = int((total_sec + 1.0) * self.sample_rate)
|
||||
midi_data = np.zeros((2, total_samples), dtype=np.float32)
|
||||
sf_total_samples = int((total_sec + 1.0) * self.sample_rate)
|
||||
midi_data = np.zeros((2, sf_total_samples), dtype=np.float32)
|
||||
for ev in midi_events:
|
||||
note = ev.get("note", 60)
|
||||
velocity = ev.get("velocity", 100)
|
||||
@@ -213,13 +254,15 @@ class PythonRenderEngine:
|
||||
block = fl.get_samples(int(dur_s)) if hasattr(fl, 'get_samples') else np.zeros((2, dur_s), dtype=np.float32)
|
||||
fl.noteoff(midi_channel, note)
|
||||
if block.shape[1] > 0:
|
||||
end_s = min(start_s + block.shape[1], total_samples)
|
||||
end_s = min(start_s + block.shape[1], sf_total_samples)
|
||||
actual = end_s - start_s
|
||||
if actual > 0:
|
||||
midi_data[:, start_s:end_s] += block[:, :actual]
|
||||
synth_buffer = midi_data
|
||||
fl.delete()
|
||||
else:
|
||||
if not HAS_PYFLUIDSYNTH:
|
||||
logger.warning("[RenderEngine] pyfluidsynth not available, falling back to oscillator synth")
|
||||
synth_buffer = render_midi_events_to_audio(
|
||||
midi_events=midi_events, sr=self.sample_rate, bpm=bpm, instrument='synth'
|
||||
)
|
||||
|
||||
+16
-3
@@ -6492,7 +6492,17 @@ const App = () => {
|
||||
if (t.id !== trackId) return t;
|
||||
const hasInstrument = !!instrumentId;
|
||||
const isSfInstrument = instrumentId && typeof instrumentId === 'string' && instrumentId.startsWith('sf_');
|
||||
return { ...t, instrumentId, instrumentProgram: programNumber !== undefined ? programNumber : undefined, instrumentName: displayName, soundfont_bank: bankNumber !== undefined ? bankNumber : (isSfInstrument ? 0 : undefined), soundfont_program: programNumber !== undefined ? programNumber : undefined, type: hasInstrument ? 'MIDI' : (t.type === 'MIDI' ? 'audio' : t.type) };
|
||||
const sfBank = bankNumber !== undefined ? bankNumber : (isSfInstrument ? 0 : undefined);
|
||||
const sfProg = programNumber !== undefined ? programNumber : undefined;
|
||||
const instrType = isSfInstrument ? 'soundfont' : (hasInstrument ? 'vst3' : 'default');
|
||||
const synthEngine = hasInstrument ? {
|
||||
type: instrType,
|
||||
plugin_id: instrumentId,
|
||||
soundfont_bank: sfBank !== undefined ? sfBank : 0,
|
||||
soundfont_program: sfProg !== undefined ? sfProg : 0,
|
||||
soundfont_id: isSfInstrument ? instrumentId.replace('sf_', '') : ''
|
||||
} : undefined;
|
||||
return { ...t, instrumentId, instrumentProgram: sfProg, instrumentName: displayName, soundfont_bank: sfBank, soundfont_program: sfProg, synth_engine: synthEngine, type: hasInstrument ? 'MIDI' : (t.type === 'MIDI' ? 'audio' : t.type) };
|
||||
}));
|
||||
setInstrumentDropdownTrackId(null);
|
||||
setInstrumentDropdownBtnRect(null);
|
||||
@@ -6544,7 +6554,9 @@ const App = () => {
|
||||
// Set instrument on track immediately so Synth button shows the name
|
||||
updateActiveTracks(prev => prev.map(t => {
|
||||
if (t.id !== trackId) return t;
|
||||
return { ...t, instrumentId, instrumentProgram: undefined, instrumentName: displayName };
|
||||
const sfClean = instrumentId.replace('sf_', '');
|
||||
const synthEngine = { type: 'soundfont', plugin_id: instrumentId, soundfont_bank: 0, soundfont_program: 0, soundfont_id: sfClean };
|
||||
return { ...t, instrumentId, instrumentProgram: undefined, instrumentName: displayName, synth_engine: synthEngine };
|
||||
}));
|
||||
setSelectedSoundFontId(instrumentId);
|
||||
setSynthCategory('soundfont');
|
||||
@@ -10053,6 +10065,7 @@ const App = () => {
|
||||
const track = activeTracksRef.current ? activeTracksRef.current.find(t => t.id === st.trackId) : null;
|
||||
const destNode = getOrCreateTrackNode(track, context);
|
||||
const instrumentProgram = track ? track.instrumentProgram : undefined;
|
||||
const synthEngine = track ? track.synth_engine : undefined;
|
||||
midiNotes.forEach(note => {
|
||||
const noteOnBeat = note.start_beat || 0;
|
||||
const noteDurBeat = note.duration_beats || 1;
|
||||
@@ -10064,7 +10077,7 @@ const App = () => {
|
||||
const scheduledTime = startWallTime + effectiveStart;
|
||||
const durMs = effectiveDur * 1000;
|
||||
if (window.SonicSF) {
|
||||
window.SonicSF.playNote(note.pitch || 60, note.velocity || 0.8, durMs, scheduledTime, instrumentProgram, destNode);
|
||||
window.SonicSF.playNote(note.pitch || 60, note.velocity || 0.8, durMs, scheduledTime, instrumentProgram, destNode, undefined, synthEngine);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -218,6 +218,7 @@ Ví dụ: "Hãy chọn và copy từ bar 4 đến bar 12 của track 1 sau đó
|
||||
function buildAIPromptContext(dawState) {
|
||||
const tracks = (dawState.tracks || []).map(t => {
|
||||
const clips = t.clips && t.clips.length > 0 ? t.clips : (t.buffer ? [{ id: 'default_' + t.id, name: t.name, startTime: t.startTime || 0, duration: t.buffer.duration }] : []);
|
||||
const se = t.synth_engine || null;
|
||||
return {
|
||||
id: t.id,
|
||||
name: t.name,
|
||||
@@ -227,6 +228,7 @@ Ví dụ: "Hãy chọn và copy từ bar 4 đến bar 12 của track 1 sau đó
|
||||
solo: t.solo,
|
||||
volumeDb: t.volumeDb ?? 0,
|
||||
pan: t.pan ?? 0,
|
||||
synth_engine: se ? { type: se.type, plugin_id: se.plugin_id, soundfont_bank: se.soundfont_bank, soundfont_program: se.soundfont_program, soundfont_id: se.soundfont_id } : undefined,
|
||||
clips: clips.map(c => ({ id: c.id, name: c.name, startTime: parseFloat((c.startTime || 0).toFixed(3)), duration: parseFloat((c.buffer ? c.buffer.duration : 0).toFixed(3)) }))
|
||||
};
|
||||
});
|
||||
|
||||
@@ -57,7 +57,11 @@
|
||||
return ch;
|
||||
},
|
||||
|
||||
applyAITrackInstrument: function (bank, program) {
|
||||
applyAITrackInstrument: function (bank, program, synthEngine) {
|
||||
if (synthEngine) {
|
||||
bank = bank !== undefined ? bank : (synthEngine.soundfont_bank || 0);
|
||||
program = program !== undefined ? program : (synthEngine.soundfont_program || 0);
|
||||
}
|
||||
const channel = this.allocateChannel(bank);
|
||||
this.controllerChange(channel, 0, bank);
|
||||
this.programChange(channel, program);
|
||||
@@ -79,11 +83,20 @@
|
||||
return buffer;
|
||||
},
|
||||
|
||||
playNote: function (note, velocity, durationMs, startTime, program, destinationNode, channel) {
|
||||
playNote: function (note, velocity, durationMs, startTime, program, destinationNode, channel, synthEngine) {
|
||||
const ctx = getCtx();
|
||||
const freq = 440 * Math.pow(2, (note - 69) / 12);
|
||||
if (freq <= 0 || isNaN(freq)) return null;
|
||||
|
||||
// Apply synth_engine state if provided
|
||||
if (synthEngine) {
|
||||
const ch = channel !== undefined ? channel : (synthEngine.soundfont_bank === 128 ? 9 : 0);
|
||||
this.controllerChange(ch, 0, synthEngine.soundfont_bank || 0);
|
||||
this.programChange(ch, synthEngine.soundfont_program || 0);
|
||||
if (channel === undefined) channel = ch;
|
||||
if (program === undefined) program = synthEngine.soundfont_program;
|
||||
}
|
||||
|
||||
const osc = ctx.createOscillator();
|
||||
const noteGain = ctx.createGain();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user