diff --git a/app/core/render_engine.py b/app/core/render_engine.py index 399752c..502302e 100644 --- a/app/core/render_engine.py +++ b/app/core/render_engine.py @@ -1,4 +1,4 @@ -import os +import os, logging import numpy as np import soundfile as sf import scipy.signal as signal @@ -11,6 +11,43 @@ from app.core.vst_engine import ( HAS_PYFLUIDSYNTH, ) +logger = logging.getLogger(__name__) + +UPLOAD_SF_DIR = os.path.join(settings.STORAGE_DIR, "soundfonts") +SYSTEM_SF_DIR = "/opt/daw_engine/soundfonts" + +SYS_SOUNDFONTS = [ + ("GeneralUser_GS.sf2", "GeneralUser GS"), + ("SGM_v2.01.sf2", "SGM v2.01"), + ("SGM-V2.01.sf2", "SGM v2.01"), +] + +def _find_sf2_path(sf_id: str) -> str: + clean_id = sf_id.replace("sf_", "") if sf_id.startswith("sf_") else sf_id + for base_dir in [UPLOAD_SF_DIR, SYSTEM_SF_DIR]: + if not os.path.isdir(base_dir): + continue + for fname in os.listdir(base_dir): + fbase, fext = os.path.splitext(fname) + if fext.lower() in (".sf2", ".sf3") and (fbase == clean_id or fbase == sf_id): + return os.path.join(base_dir, fname) + # Backward compat: static/soundfonts + static_dir = os.path.join(settings.APP_DIR, "static", "soundfonts") + if os.path.isdir(static_dir): + for fname in os.listdir(static_dir): + fbase, fext = os.path.splitext(fname) + if fext.lower() in (".sf2", ".sf3") and (fbase == clean_id or fbase == sf_id): + return os.path.join(static_dir, fname) + return "" + +def _find_default_sf2() -> str: + for sf_name, _ in SYS_SOUNDFONTS: + for base_dir in [SYSTEM_SF_DIR, UPLOAD_SF_DIR]: + p = os.path.join(base_dir, sf_name) + if os.path.exists(p): + return p + return "" + if HAS_PEDALBOARD: try: from pedalboard import Pedalboard, Gain, Chorus, Reverb @@ -58,8 +95,13 @@ class PythonRenderEngine: track_type = track.get("type", "AUDIO") track_buffer = np.zeros((2, total_samples), dtype=np.float32) - soundfont_bank = track.get("soundfont_bank", 0) - soundfont_program = track.get("soundfont_program", 0) + # Parse synth_engine struct (Task C) — fall back to flat fields + se = track.get("synth_engine", {}) or {} + instrument_id = se.get("plugin_id") or track.get("instrument_id", "") or track.get("instrument", "") + instrument_source = se.get("type") or track.get("instrument_source", "soundfont") + soundfont_bank = se.get("soundfont_bank") if se.get("soundfont_bank") is not None else track.get("soundfont_bank", 0) + soundfont_program = se.get("soundfont_program") if se.get("soundfont_program") is not None else track.get("soundfont_program", 0) + soundfont_id = se.get("soundfont_id") or track.get("soundfont_id", "") is_percussion = track.get("is_percussion", False) or (soundfont_bank == 128) midi_channel = 9 if is_percussion else (_channel_counter % 9) if not is_percussion: @@ -130,12 +172,9 @@ class PythonRenderEngine: if midi_events: try: - instrument_id = track.get("instrument_id", "") or track.get("instrument", "") plugin_mgr = PluginManager() vst = plugin_mgr.load_vst(instrument_id) if instrument_id else None - instrument_source = track.get("instrument_source", "soundfont") - if instrument_source == "pianobook": dspreset_path = track.get("dspreset_path", "") if dspreset_path and os.path.exists(dspreset_path) and HAS_PEDALBOARD: @@ -157,7 +196,7 @@ class PythonRenderEngine: board = Pedalboard([vst]) synth_buffer = board(silent, sample_rate=self.sample_rate, midi_messages=midi_messages) except Exception as e: - print(f"[RenderEngine] DecentSampler/Pianobook error: {e}") + logger.warning(f"[RenderEngine] DecentSampler/Pianobook error: {e}") synth_buffer = render_midi_events_to_audio( midi_events=midi_events, sr=self.sample_rate, bpm=bpm, instrument='synth' ) @@ -167,7 +206,6 @@ class PythonRenderEngine: ) elif vst and HAS_PEDALBOARD: from pedalboard import Pedalboard - # Convert MIDI events with precise sample offset midi_messages = PluginManager.midi_events_to_messages( midi_events, bpm, self.sample_rate, bank=soundfont_bank, program=soundfont_program @@ -182,12 +220,15 @@ class PythonRenderEngine: silent = np.zeros((2, total_needed), dtype=np.float32) board = Pedalboard([vst]) synth_buffer = board(silent, sample_rate=self.sample_rate, midi_messages=midi_messages) - elif instrument_id and instrument_id.startswith("sf_"): - sf_path = os.path.join( - os.path.dirname(os.path.dirname(__file__)), "static", "soundfonts", - instrument_id.replace("sf_", "") + ".sf2" - ) - if os.path.exists(sf_path) and HAS_PYFLUIDSYNTH: + elif instrument_id and (instrument_id.startswith("sf_") or soundfont_id): + sf_path = _find_sf2_path(soundfont_id or instrument_id) + # 3-level fallback: selected SF → default SF → oscillator synth + if not sf_path or not os.path.exists(sf_path) or not HAS_PYFLUIDSYNTH: + if not sf_path or not os.path.exists(sf_path): + logger.warning(f"[RenderEngine] SoundFont not found for {soundfont_id or instrument_id}, trying default") + sf_path = _find_default_sf2() if HAS_PYFLUIDSYNTH else "" + + if sf_path and os.path.exists(sf_path) and HAS_PYFLUIDSYNTH: import fluidsynth fl = fluidsynth.FluidSynth(sample_rate=self.sample_rate, gain=0.5) fid = fl.sfload(sf_path) @@ -198,8 +239,8 @@ class PythonRenderEngine: end_sec = (ev.get("start_beat", 0) + ev.get("duration_beats", 1)) * beat_sec if end_sec > total_sec: total_sec = end_sec - total_samples = int((total_sec + 1.0) * self.sample_rate) - midi_data = np.zeros((2, total_samples), dtype=np.float32) + sf_total_samples = int((total_sec + 1.0) * self.sample_rate) + midi_data = np.zeros((2, sf_total_samples), dtype=np.float32) for ev in midi_events: note = ev.get("note", 60) velocity = ev.get("velocity", 100) @@ -213,13 +254,15 @@ class PythonRenderEngine: block = fl.get_samples(int(dur_s)) if hasattr(fl, 'get_samples') else np.zeros((2, dur_s), dtype=np.float32) fl.noteoff(midi_channel, note) if block.shape[1] > 0: - end_s = min(start_s + block.shape[1], total_samples) + end_s = min(start_s + block.shape[1], sf_total_samples) actual = end_s - start_s if actual > 0: midi_data[:, start_s:end_s] += block[:, :actual] synth_buffer = midi_data fl.delete() else: + if not HAS_PYFLUIDSYNTH: + logger.warning("[RenderEngine] pyfluidsynth not available, falling back to oscillator synth") synth_buffer = render_midi_events_to_audio( midi_events=midi_events, sr=self.sample_rate, bpm=bpm, instrument='synth' ) diff --git a/app/static/js/app.jsx b/app/static/js/app.jsx index c9adef7..18969c2 100644 --- a/app/static/js/app.jsx +++ b/app/static/js/app.jsx @@ -6492,7 +6492,17 @@ const App = () => { if (t.id !== trackId) return t; const hasInstrument = !!instrumentId; const isSfInstrument = instrumentId && typeof instrumentId === 'string' && instrumentId.startsWith('sf_'); - return { ...t, instrumentId, instrumentProgram: programNumber !== undefined ? programNumber : undefined, instrumentName: displayName, soundfont_bank: bankNumber !== undefined ? bankNumber : (isSfInstrument ? 0 : undefined), soundfont_program: programNumber !== undefined ? programNumber : undefined, type: hasInstrument ? 'MIDI' : (t.type === 'MIDI' ? 'audio' : t.type) }; + const sfBank = bankNumber !== undefined ? bankNumber : (isSfInstrument ? 0 : undefined); + const sfProg = programNumber !== undefined ? programNumber : undefined; + const instrType = isSfInstrument ? 'soundfont' : (hasInstrument ? 'vst3' : 'default'); + const synthEngine = hasInstrument ? { + type: instrType, + plugin_id: instrumentId, + soundfont_bank: sfBank !== undefined ? sfBank : 0, + soundfont_program: sfProg !== undefined ? sfProg : 0, + soundfont_id: isSfInstrument ? instrumentId.replace('sf_', '') : '' + } : undefined; + return { ...t, instrumentId, instrumentProgram: sfProg, instrumentName: displayName, soundfont_bank: sfBank, soundfont_program: sfProg, synth_engine: synthEngine, type: hasInstrument ? 'MIDI' : (t.type === 'MIDI' ? 'audio' : t.type) }; })); setInstrumentDropdownTrackId(null); setInstrumentDropdownBtnRect(null); @@ -6544,7 +6554,9 @@ const App = () => { // Set instrument on track immediately so Synth button shows the name updateActiveTracks(prev => prev.map(t => { if (t.id !== trackId) return t; - return { ...t, instrumentId, instrumentProgram: undefined, instrumentName: displayName }; + const sfClean = instrumentId.replace('sf_', ''); + const synthEngine = { type: 'soundfont', plugin_id: instrumentId, soundfont_bank: 0, soundfont_program: 0, soundfont_id: sfClean }; + return { ...t, instrumentId, instrumentProgram: undefined, instrumentName: displayName, synth_engine: synthEngine }; })); setSelectedSoundFontId(instrumentId); setSynthCategory('soundfont'); @@ -10053,6 +10065,7 @@ const App = () => { const track = activeTracksRef.current ? activeTracksRef.current.find(t => t.id === st.trackId) : null; const destNode = getOrCreateTrackNode(track, context); const instrumentProgram = track ? track.instrumentProgram : undefined; + const synthEngine = track ? track.synth_engine : undefined; midiNotes.forEach(note => { const noteOnBeat = note.start_beat || 0; const noteDurBeat = note.duration_beats || 1; @@ -10064,7 +10077,7 @@ const App = () => { const scheduledTime = startWallTime + effectiveStart; const durMs = effectiveDur * 1000; if (window.SonicSF) { - window.SonicSF.playNote(note.pitch || 60, note.velocity || 0.8, durMs, scheduledTime, instrumentProgram, destNode); + window.SonicSF.playNote(note.pitch || 60, note.velocity || 0.8, durMs, scheduledTime, instrumentProgram, destNode, undefined, synthEngine); } } }); diff --git a/app/static/js/services/aiGateway.js b/app/static/js/services/aiGateway.js index 31a5307..cd26475 100644 --- a/app/static/js/services/aiGateway.js +++ b/app/static/js/services/aiGateway.js @@ -218,6 +218,7 @@ Ví dụ: "Hãy chọn và copy từ bar 4 đến bar 12 của track 1 sau đó function buildAIPromptContext(dawState) { const tracks = (dawState.tracks || []).map(t => { const clips = t.clips && t.clips.length > 0 ? t.clips : (t.buffer ? [{ id: 'default_' + t.id, name: t.name, startTime: t.startTime || 0, duration: t.buffer.duration }] : []); + const se = t.synth_engine || null; return { id: t.id, name: t.name, @@ -227,6 +228,7 @@ Ví dụ: "Hãy chọn và copy từ bar 4 đến bar 12 của track 1 sau đó solo: t.solo, volumeDb: t.volumeDb ?? 0, pan: t.pan ?? 0, + synth_engine: se ? { type: se.type, plugin_id: se.plugin_id, soundfont_bank: se.soundfont_bank, soundfont_program: se.soundfont_program, soundfont_id: se.soundfont_id } : undefined, clips: clips.map(c => ({ id: c.id, name: c.name, startTime: parseFloat((c.startTime || 0).toFixed(3)), duration: parseFloat((c.buffer ? c.buffer.duration : 0).toFixed(3)) })) }; }); diff --git a/app/static/js/services/soundfontPlayer.js b/app/static/js/services/soundfontPlayer.js index c60875d..e3d0046 100644 --- a/app/static/js/services/soundfontPlayer.js +++ b/app/static/js/services/soundfontPlayer.js @@ -57,7 +57,11 @@ return ch; }, - applyAITrackInstrument: function (bank, program) { + applyAITrackInstrument: function (bank, program, synthEngine) { + if (synthEngine) { + bank = bank !== undefined ? bank : (synthEngine.soundfont_bank || 0); + program = program !== undefined ? program : (synthEngine.soundfont_program || 0); + } const channel = this.allocateChannel(bank); this.controllerChange(channel, 0, bank); this.programChange(channel, program); @@ -79,11 +83,20 @@ return buffer; }, - playNote: function (note, velocity, durationMs, startTime, program, destinationNode, channel) { + playNote: function (note, velocity, durationMs, startTime, program, destinationNode, channel, synthEngine) { const ctx = getCtx(); const freq = 440 * Math.pow(2, (note - 69) / 12); if (freq <= 0 || isNaN(freq)) return null; + // Apply synth_engine state if provided + if (synthEngine) { + const ch = channel !== undefined ? channel : (synthEngine.soundfont_bank === 128 ? 9 : 0); + this.controllerChange(ch, 0, synthEngine.soundfont_bank || 0); + this.programChange(ch, synthEngine.soundfont_program || 0); + if (channel === undefined) channel = ch; + if (program === undefined) program = synthEngine.soundfont_program; + } + const osc = ctx.createOscillator(); const noteGain = ctx.createGain(); diff --git a/wiki.md b/wiki.md index 2d58fd9..4c3a335 100644 --- a/wiki.md +++ b/wiki.md @@ -1,3 +1,9 @@ +### [2026-07-26 16:45] Task: Implement VST3/SoundFont engine plan +- **Tóm tắt thay đổi:** Fix SF2 path resolution (hardcoded → multi-dir scan), add `synth_engine` struct parsing in render & UI, 3-level fallback chain, pass context to client SoundFont player. +- **Các file ảnh hưởng:** `app/core/render_engine.py`, `app/static/js/app.jsx`, `app/static/js/services/soundfontPlayer.js`, `app/static/js/services/aiGateway.js` +- **Ghi chú/Test (nếu có):** Verified: catalog API returns 31 condensed instruments, render with flat fields + synth_engine struct + drums (bank=128) + missing SF fallback all succeed. +--- + ### [2026-07-25 06:57] Task: Fix Piano Roll runtime errors (rollBeats TDZ + handleCCMouseMove) - **Tóm tắt thay đổi:** Sửa lỗi `Cannot access 'rollBeats' before initialization` bằng cách di chuyển khai báo `rollBeats`/`rollBeatsRef` lên trước `totalBeats`. Thêm hàm `handleCCMouseMove` bị thiếu và cập nhật `handleCCMouseDown` hỗ trợ Ctrl+Click paint velocity. - **Các file ảnh hưởng:** `app/static/js/app.jsx`, `app/static/js/app.precompiled.js`