From efe26e3277f5e08e817ce00c030949549f89a9d0 Mon Sep 17 00:00:00 2001 From: locphamtran Date: Mon, 17 Aug 2026 11:10:43 +0700 Subject: [PATCH] =?UTF-8?q?phase3:=20render=5Fengine=20export=20qua=20nati?= =?UTF-8?q?ve=5Fbridge,=20FX=20scipy=20fallback,=20b=E1=BB=8F=20pedalboard?= =?UTF-8?q?=20branch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - VSTi track: SF_RENDER_ENGINE=bridge → native_render.render_offline (velocity float 0..1 từ midi_events MIDI 0..127), fallback oscillator khi lỗi - chorus/reverb/gain: luôn dùng scipy/numpy fallback, xóa nhánh HAS_PEDALBOARD - bỏ top-level import pedalboard (Pedalboard/Gain/Chorus/Reverb) - test mới: bridge branch fallback, FX chorus/reverb scipy --- app/core/render_engine.py | 200 +++++++++++++++++++----------------- tests/test_render_engine.py | 69 +++++++++++++ 2 files changed, 175 insertions(+), 94 deletions(-) diff --git a/app/core/render_engine.py b/app/core/render_engine.py index 8984043..2fa383c 100644 --- a/app/core/render_engine.py +++ b/app/core/render_engine.py @@ -67,12 +67,6 @@ def _find_default_sf2() -> str: return p return "" -if HAS_PEDALBOARD: - try: - from pedalboard import Pedalboard, Gain, Chorus, Reverb - except Exception: - HAS_PEDALBOARD = False - class PythonRenderEngine: @@ -239,38 +233,80 @@ class PythonRenderEngine: synth_buffer = render_midi_events_to_audio( midi_events=midi_events, sr=self.sample_rate, bpm=bpm, instrument='synth' ) - elif vst and HAS_PEDALBOARD: - from pedalboard import Pedalboard - # Preset bridge Carla → pedalboard: preset_id - # (thư viện storage/presets), preset_path (file) - # hoặc preset_data (base64 .vstpreset nhúng trong - # project). Cùng sample rate → âm render = âm đã - # chỉnh trong Carla. - try: - se2 = track.get("synth_engine", {}) or {} - apply_preset_to_plugin( - vst, - preset_id=se2.get("preset_id") or track.get("preset_id"), - preset_path=se2.get("preset_path") or track.get("preset_path"), - preset_data_b64=se2.get("preset_data") or track.get("preset_data"), + elif instrument_id and not instrument_id.startswith("sf_"): + # VSTi track: native_bridge --render (SF_RENDER_ENGINE=bridge) + # thay pedalboard; ngược lại giữ pedalboard path cũ. + if settings.RENDER_ENGINE == "bridge": + try: + from app.core import native_render + se2 = track.get("synth_engine", {}) or {} + bridge_notes = [{ + "pitch": ev["note"], + "velocity": float(ev["velocity"]) / 127.0, + "start_beat": ev["start_beat"], + "duration_beats": ev["duration_beats"], + } for ev in midi_events] + out_path, _dur = native_render.render_offline( + instrument_id=instrument_id, + notes=bridge_notes, + bpm=bpm, + sample_rate=self.sample_rate, + preset_id=se2.get("preset_id") or track.get("preset_id"), + preset_path=se2.get("preset_path") or track.get("preset_path"), + preset_data_b64=se2.get("preset_data") or track.get("preset_data"), + soundfont_bank=soundfont_bank, + soundfont_program=soundfont_program, + ) + synth_buffer, _ = sf.read(out_path, dtype="float32") + if synth_buffer.ndim == 1: + synth_buffer = np.vstack([synth_buffer, synth_buffer]) + else: + synth_buffer = synth_buffer.T + try: + os.remove(out_path) + except Exception: + pass + except Exception as e: + logger.warning("[RenderEngine] native_bridge render failed: %s", e) + synth_buffer = render_midi_events_to_audio( + midi_events=midi_events, sr=self.sample_rate, bpm=bpm, instrument='synth' + ) + elif vst and HAS_PEDALBOARD: + from pedalboard import Pedalboard + # Preset bridge Carla → pedalboard: preset_id + # (thư viện storage/presets), preset_path (file) + # hoặc preset_data (base64 .vstpreset nhúng trong + # project). Cùng sample rate → âm render = âm đã + # chỉnh trong Carla. + try: + se2 = track.get("synth_engine", {}) or {} + apply_preset_to_plugin( + vst, + preset_id=se2.get("preset_id") or track.get("preset_id"), + preset_path=se2.get("preset_path") or track.get("preset_path"), + preset_data_b64=se2.get("preset_data") or track.get("preset_data"), + ) + except Exception as e: + logger.warning("[RenderEngine] Preset apply failed: %s", e) + midi_messages = PluginManager.midi_events_to_messages( + midi_events, bpm, self.sample_rate, + bank=soundfont_bank, program=soundfont_program + ) + total_needed = 0 + for ev in midi_events: + end_sec = (ev.get("start_beat", 0) + ev.get("duration_beats", 1)) * (60.0 / bpm) + dur_samples = int(end_sec * self.sample_rate) + if dur_samples > total_needed: + total_needed = dur_samples + total_needed = max(total_needed, 1024) + silent = np.zeros((2, total_needed), dtype=np.float32) + # pedalboard >= 0.9: instrument KHÔNG vào Pedalboard container + synth_buffer = vst(midi_messages, sample_rate=self.sample_rate, + duration=total_needed / float(self.sample_rate), num_channels=2) + else: + synth_buffer = render_midi_events_to_audio( + midi_events=midi_events, sr=self.sample_rate, bpm=bpm, instrument='synth' ) - except Exception as e: - logger.warning("[RenderEngine] Preset apply failed: %s", e) - midi_messages = PluginManager.midi_events_to_messages( - midi_events, bpm, self.sample_rate, - bank=soundfont_bank, program=soundfont_program - ) - total_needed = 0 - for ev in midi_events: - end_sec = (ev.get("start_beat", 0) + ev.get("duration_beats", 1)) * (60.0 / bpm) - dur_samples = int(end_sec * self.sample_rate) - if dur_samples > total_needed: - total_needed = dur_samples - total_needed = max(total_needed, 1024) - silent = np.zeros((2, total_needed), dtype=np.float32) - # pedalboard >= 0.9: instrument KHÔNG vào Pedalboard container - synth_buffer = vst(midi_messages, sample_rate=self.sample_rate, - duration=total_needed / float(self.sample_rate), num_channels=2) elif soundfont_id or (instrument_id and instrument_id.startswith("sf_")): sf_path = _find_sf2_path(soundfont_id or instrument_id) # 3-level fallback: selected SF → default SF → oscillator synth @@ -378,66 +414,42 @@ class PythonRenderEngine: # Apply Track FX (Chorus or Reverb) fx_type = track.get("fx_type") if fx_type == "chorus": - if HAS_PEDALBOARD: - try: - board = Pedalboard([Chorus(rate_hz=1.5, depth=0.25)]) - track_buffer = board(track_buffer, sample_rate=self.sample_rate) - except Exception as e: - logger.warning("[RenderEngine] Pedalboard Chorus failed: %s", e) - else: - # Fallback chorus using simple LFO delay modulation in scipy/numpy - try: - # 1.5 Hz sine LFO, modulating delay time between 15ms and 25ms (average 20ms) - lfo = 0.020 + 0.005 * np.sin(2 * np.pi * 1.5 * np.arange(total_samples) / self.sample_rate) - dry = track_buffer * 0.6 - wet = np.zeros_like(track_buffer) - for ch in range(2): - indices = np.arange(total_samples) - (lfo * self.sample_rate) - indices = np.clip(indices, 0, total_samples - 1).astype(np.int32) - wet[ch, :] = track_buffer[ch, indices] - track_buffer = dry + wet * 0.5 - except Exception as e: - logger.warning("[RenderEngine] Fallback Chorus failed: %s", e) + # Chorus: LFO delay modulation (scipy/numpy fallback — thay + # pedalboard GPL-3.0; chất lượng thấp hơn, upgrade native DSP sau). + try: + lfo = 0.020 + 0.005 * np.sin(2 * np.pi * 1.5 * np.arange(total_samples) / self.sample_rate) + dry = track_buffer * 0.6 + wet = np.zeros_like(track_buffer) + for ch in range(2): + indices = np.arange(total_samples) - (lfo * self.sample_rate) + indices = np.clip(indices, 0, total_samples - 1).astype(np.int32) + wet[ch, :] = track_buffer[ch, indices] + track_buffer = dry + wet * 0.5 + except Exception as e: + logger.warning("[RenderEngine] Fallback Chorus failed: %s", e) elif fx_type == "reverb": - if HAS_PEDALBOARD: - try: - board = Pedalboard([Reverb(room_size=0.5, wet_level=0.4, dry_level=0.6)]) - track_buffer = board(track_buffer, sample_rate=self.sample_rate) - except Exception as e: - logger.warning("[RenderEngine] Pedalboard Reverb failed: %s", e) - else: - # Fallback reverb using exponentially decaying noise room impulse response - try: - # Generate impulse response (decaying noise) - len_ir = int(self.sample_rate * 2.0) - t_ir = np.arange(len_ir) / self.sample_rate - decay = np.exp(-t_ir / 0.5) - ir_l = (np.random.rand(len_ir) * 2 - 1) * decay - ir_r = (np.random.rand(len_ir) * 2 - 1) * decay - - dry = track_buffer * 0.6 - wet = np.zeros_like(track_buffer) - for ch in range(2): - ir = ir_l if ch == 0 else ir_r - # Convolve - from scipy.signal import convolve - conv = convolve(track_buffer[ch, :], ir, mode='full')[:total_samples] - wet[ch, :] = conv - track_buffer = dry + wet * 0.4 - except Exception as e: - logger.warning("[RenderEngine] Fallback Reverb failed: %s", e) + # Reverb: decaying-noise IR convolution (scipy — thay pedalboard GPL). + try: + len_ir = int(self.sample_rate * 2.0) + t_ir = np.arange(len_ir) / self.sample_rate + decay = np.exp(-t_ir / 0.5) + ir_l = (np.random.rand(len_ir) * 2 - 1) * decay + ir_r = (np.random.rand(len_ir) * 2 - 1) * decay + + dry = track_buffer * 0.6 + wet = np.zeros_like(track_buffer) + for ch in range(2): + ir = ir_l if ch == 0 else ir_r + from scipy.signal import convolve + conv = convolve(track_buffer[ch, :], ir, mode='full')[:total_samples] + wet[ch, :] = conv + track_buffer = dry + wet * 0.4 + except Exception as e: + logger.warning("[RenderEngine] Fallback Reverb failed: %s", e) # Process track volume - if HAS_PEDALBOARD: - try: - board = Pedalboard([Gain(gain_db=vol_db)]) - processed_track = board(track_buffer, sample_rate=self.sample_rate) - except Exception: - gain_linear = 10 ** (vol_db / 20.0) - processed_track = track_buffer * gain_linear - else: - gain_linear = 10 ** (vol_db / 20.0) - processed_track = track_buffer * gain_linear + gain_linear = 10 ** (vol_db / 20.0) + processed_track = track_buffer * gain_linear # Apply Track Pan if pan != 0.0: diff --git a/tests/test_render_engine.py b/tests/test_render_engine.py index ec713eb..d6fb074 100644 --- a/tests/test_render_engine.py +++ b/tests/test_render_engine.py @@ -18,3 +18,72 @@ def test_render_session_container_empty(): buf = engine.render_session_container(session, {}, 120.0, 4, 1000) assert buf.shape == (2, 1000) assert (buf == 0.0).all() + +def _midi_track(instrument_id, fx_type=None): + return { + "id": "t1", + "type": "MIDI", + "instrument_id": instrument_id, + "instrument_source": "vsti", + "fx_type": fx_type, + "volume_db": 0.0, + "pan": 0.0, + "mute": False, + "items": [{ + "type": "MIDI_ITEM", + "start_bar": 0.0, + "duration_bars": 1.0, + "clip_start_offset_bars": 0.0, + "source_data": { + "notes": [ + {"pitch": 60, "velocity": 0.8, "start_beat": 0.0, "duration_beats": 1.0} + ] + }, + }], + } + + +def test_render_session_vsti_bridge_unknown_instrument_falls_back(monkeypatch): + """SF_RENDER_ENGINE=bridge: track VSTi đi vào native_render; instrument lạ + (404) → fallback oscillator, không crash, không cần pedalboard.""" + from app.api.v1 import plugins as pl + monkeypatch.setattr(pl.settings, "RENDER_ENGINE", "bridge") + engine = PythonRenderEngine() + session = {"tracks": [_midi_track("__no_such_plugin__")]} + buf = engine.render_session_container(session, {}, 120.0, 4, 44100) + assert buf.shape == (2, 44100) + # oscillator sinh âm thật (non-silent) + assert (buf != 0.0).any() + + +def test_render_session_vsti_pedalboard_default_falls_back(monkeypatch): + """Mặc định pedalboard: không có plugin load được → oscillator (không crash).""" + from app.api.v1 import plugins as pl + monkeypatch.setattr(pl.settings, "RENDER_ENGINE", "pedalboard") + engine = PythonRenderEngine() + session = {"tracks": [_midi_track("__no_such_plugin__")]} + buf = engine.render_session_container(session, {}, 120.0, 4, 44100) + assert buf.shape == (2, 44100) + + +def test_render_session_fx_chorus_scipy_fallback(): + """FX chorus: fallback scipy (không pedalboard) không crash, thay đổi âm.""" + from app.api.v1 import plugins as pl + pl.settings.RENDER_ENGINE = "pedalboard" + engine = PythonRenderEngine() + session = {"tracks": [_midi_track("__no_such_plugin__", fx_type="chorus")]} + buf = engine.render_session_container(session, {}, 120.0, 4, 44100) + assert buf.shape == (2, 44100) + assert (buf != 0.0).any() + + +def test_render_session_fx_reverb_scipy_fallback(): + """FX reverb: fallback scipy (không pedalboard) không crash, thay đổi âm.""" + from app.api.v1 import plugins as pl + pl.settings.RENDER_ENGINE = "pedalboard" + engine = PythonRenderEngine() + session = {"tracks": [_midi_track("__no_such_plugin__", fx_type="reverb")]} + buf = engine.render_session_container(session, {}, 120.0, 4, 44100) + assert buf.shape == (2, 44100) + assert (buf != 0.0).any() +