feat: tách luồng âm instrument theo môi trường — docker dùng FluidSynthWASM, standalone dùng native pyfluidsynth (+Carla VSTi)

- runtime: detect()/capabilities() trả environment (docker|standalone)
- vst_engine: render_soundfont_midi_to_audio (pyfluidsynth native, event-stream, release tail)
- plugins: POST /soundfont-render — render MIDI notes -> WAV (auth, fallback default SF)
- frontend: SonicRuntime.environment + dataset.environment; app.jsx route preview/transport:
  standalone+SF -> soundfont-render WAV (playNativeSfNote/scheduleNativeSfItem),
  docker giữ WASM, VSTi standalone giữ Carla bridge
This commit is contained in:
2026-08-10 20:38:54 +07:00
parent e89a594b7c
commit a0d8725541
8 changed files with 447 additions and 48 deletions
+4 -1
View File
@@ -332,10 +332,12 @@ def detect() -> dict:
platform = _detect_platform()
runtime = _detect_runtime()
carla = find_carla() if runtime == "desktop" else ""
docker = _in_docker()
return {
"platform": platform,
"runtime": runtime, # "desktop" | "headless"
"docker": _in_docker(),
"docker": docker,
"environment": "docker" if docker else "standalone", # luồng âm instrument: docker→FluidSynthWASM, standalone→native OS
"has_display": _has_display(),
"tauri_bridge": _tauri_bridge_ready(),
"carla_path": carla,
@@ -375,6 +377,7 @@ def capabilities() -> dict:
"runtime": d["runtime"],
"platform": d["platform"],
"docker": d["docker"],
"environment": d["environment"],
"features": features,
"default_dirs": {
"vst": d["default_vst_dirs"],
+68
View File
@@ -8,6 +8,74 @@ from ctypes import c_char_p
def midi_note_to_freq(note_number: int) -> float:
return 440.0 * (2.0 ** ((note_number - 69) / 12.0))
def render_soundfont_midi_to_audio(midi_events: list, sf_path: str, bank: int = 0,
program: int = 0, sr: int = 44100, bpm: float = 120.0) -> np.ndarray:
"""Native FluidSynth (pyfluidsynth) — render MIDI events bằng soundfont THẬT.
Dùng cho môi trường standalone: âm instrument xử lí trực tiếp trên OS
(backend pyfluidsynth), không phải FluidSynthWASM trong browser.
Xử lí đúng note CHỒNG nhau + đuôi release (event-stream noteon/noteoff).
Trả stereo float32 (2, N). Raise RuntimeError nếu pyfluidsynth thiếu / SF
không load được."""
import fluidsynth as _fs
settings = _fs.new_fluid_settings()
_fs.fluid_settings_setnum(settings, b'synth.sample-rate', float(sr))
synth = _fs.new_fluid_synth(settings)
try:
fid = _fs.fluid_synth_sfload(synth, os.fspath(sf_path).encode('utf-8'), 1)
if fid == -1:
raise RuntimeError(f"SoundFont load failed: {sf_path}")
_fs.fluid_synth_program_select(synth, 0, fid, int(bank), int(program))
beat_sec = 60.0 / max(30.0, bpm)
events = []
total_sec = 0.0
for ev in midi_events:
start_s = int(float(ev.get('start_beat', 0)) * beat_sec * sr)
dur_s = int(float(ev.get('duration_beats', 1)) * beat_sec * sr)
note = int(ev.get('note', 60))
vel = min(127, max(1, int(float(ev.get('velocity', 100)))))
events.append((start_s, 'on', note, vel))
events.append((start_s + dur_s, 'off', note, 0))
end_sec = (start_s + dur_s) / float(sr)
if end_sec > total_sec:
total_sec = end_sec
events.sort(key=lambda e: e[0])
total_sec = max(total_sec, 0.25) + 0.5 # đuôi reverb/release
n = int(total_sec * sr)
out = np.zeros((2, n), dtype=np.float32)
pos = 0
def _render_until(target: int):
"""Render từ pos tới target, CỘNG vào out (giữ âm đang ngân)."""
nonlocal pos
target = min(target, n)
while pos < target:
chunk = min(target - pos, 44100)
block = _fs.fluid_synth_write_s16_stereo(synth, chunk)
b = block.astype(np.float32).reshape(-1, 2).T / 32768.0
take = min(chunk, b.shape[1], target - pos)
if take > 0:
out[:, pos:pos + take] += b[:, :take]
pos += take
for t, kind, note, vel in events:
if t > pos:
_render_until(t)
if pos >= n:
break
if kind == 'on':
_fs.fluid_synth_noteon(synth, 0, note, vel)
else:
_fs.fluid_synth_noteoff(synth, 0, note)
_render_until(n) # đuôi release của notes cuối
return out
finally:
try:
_fs.delete_fluid_synth(synth)
except Exception:
pass
def render_midi_events_to_audio(midi_events: list, sr: int = 44100, bpm: float = 120.0, instrument: str = 'synth') -> np.ndarray:
beat_duration_sec = 60.0 / max(30.0, bpm)
max_duration_sec = 2.0