feat: VSTi live playback 100% client-side (không Carla) — autosample SF2 backend (pedalboard) → FluidSynth WASM qua masterBus → mastering → main out; SonicVstiAutosample ensure+dedup+cooldown; runtime gating __enableCarlaLivePlayback; âm bắt qua ensureMidiCapture → clientSideExport
This commit is contained in:
@@ -1118,6 +1118,100 @@ async def soundfont_render(req: SoundfontRenderRequest, current_user: dict = Dep
|
||||
raise HTTPException(status_code=500, detail=f"Render soundfont thất bại: {e}")
|
||||
|
||||
|
||||
class AutosampleRequest(BaseModel):
|
||||
"""Auto-sample VSTi preset -> SF2 (client FluidSynth WASM live playback).
|
||||
|
||||
Server goi tools.autosample_vsti.autosample_sf2 (pedalboard render tung
|
||||
note voi DUNG plugin + preset nhu export) -> luu {uuid}.sf2 vao
|
||||
UPLOAD_SF_DIR + .meta -> client tai qua /soundfonts/download/{uuid}.
|
||||
SF2 only (SF3 can ffmpeg/libvorbis; client WASM khong decode SF3)."""
|
||||
instrument_id: str
|
||||
preset_id: Optional[str] = None
|
||||
preset_path: Optional[str] = None
|
||||
preset_data: Optional[str] = None # base64 bytes .vstpreset (project nhung)
|
||||
low: int = 36
|
||||
high: int = 96
|
||||
step: int = 2
|
||||
duration: float = 2.5
|
||||
release: float = 1.0
|
||||
velocity: int = 100
|
||||
sample_rate: int = 44100
|
||||
|
||||
@router.post("/autosample")
|
||||
async def autosample_vsti(req: AutosampleRequest, current_user: dict = Depends(get_current_user)):
|
||||
"""Auto-sample VSTi -> SF2 cho live playback client-side (FluidSynth WASM).
|
||||
|
||||
Tra sf_id (bare uuid) de client tai SF2 va phat qua SonicSF nhu soundfont
|
||||
thuong. Chi phi autosample mot lan -> cache o client (localStorage)."""
|
||||
enforce_password_changed(current_user)
|
||||
if not HAS_PEDALBOARD:
|
||||
raise HTTPException(status_code=501, detail="pedalboard khong kha dung tren may nay")
|
||||
|
||||
# tools/ nam ngoai package app - import voi fallback path
|
||||
try:
|
||||
from tools.autosample_vsti import autosample_sf2 as _autosample_sf2
|
||||
except ImportError:
|
||||
_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
if _root not in sys.path:
|
||||
sys.path.insert(0, _root)
|
||||
from tools.autosample_vsti import autosample_sf2 as _autosample_sf2
|
||||
|
||||
file_uuid = str(uuid.uuid4())
|
||||
dest_path = os.path.join(UPLOAD_SF_DIR, file_uuid + ".sf2")
|
||||
log_lines = []
|
||||
|
||||
def _log(msg):
|
||||
log_lines.append(msg)
|
||||
print(f"[autosample] {msg}")
|
||||
|
||||
try:
|
||||
result = _autosample_sf2(
|
||||
req.instrument_id, dest_path,
|
||||
preset_id=req.preset_id,
|
||||
preset_path=req.preset_path,
|
||||
preset_data_b64=req.preset_data,
|
||||
low=req.low, high=req.high, step=req.step,
|
||||
duration=req.duration, release=req.release,
|
||||
velocity=req.velocity, sample_rate=req.sample_rate,
|
||||
log=_log,
|
||||
)
|
||||
except FileNotFoundError as e:
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
except RuntimeError as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
except Exception as e:
|
||||
try:
|
||||
if os.path.exists(dest_path):
|
||||
os.remove(dest_path)
|
||||
except Exception:
|
||||
pass
|
||||
raise HTTPException(status_code=500, detail=f"Auto-sample that bai: {e}")
|
||||
|
||||
meta_path = os.path.join(UPLOAD_SF_DIR, file_uuid + ".meta")
|
||||
try:
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
json.dump({"original_name": f"autosample_{file_uuid[:8]}.sf2",
|
||||
"uuid": file_uuid, "file": file_uuid + ".sf2",
|
||||
"plugin_id": req.instrument_id, "kind": "autosampled"}, f)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
get_scanner().scan_once()
|
||||
except Exception as e:
|
||||
print(f"[autosample] scan_once failed: {e}")
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"sf_id": file_uuid,
|
||||
"file": file_uuid + ".sf2",
|
||||
"name": f"autosample_{file_uuid[:8]}.sf2",
|
||||
"url": f"/api/v1/plugins/soundfonts/download/{file_uuid}",
|
||||
"size_bytes": result["size_bytes"],
|
||||
"note_count": result["note_count"],
|
||||
"plugin_id": req.instrument_id,
|
||||
}
|
||||
|
||||
class CarlaPlayNotesRequest(BaseModel):
|
||||
"""Phát dãy MIDI notes qua Carla bridge (OSC, realtime) — preview khi
|
||||
pedalboard không render được plugin (VD VST2). Cần Carla đang chạy với
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
"""apply_mastering — offline mastering chain, mirror chính xác client WebAudio
|
||||
(app.jsx initMasterBus / applyMasteringSettings / rebuildMasteringGraph).
|
||||
|
||||
Render export (pedalboard VSTi / soundfont) chạy qua ĐÚNG chain như live
|
||||
playback: thứ tự module từ settings['chain'] (mặc định EQ -> Imager ->
|
||||
Maximizer), mỗi module bật/tắt theo settings tương ứng. Bỏ oversample
|
||||
(WaveShaper '2x'/'4x') — offline render không cần anti-alias.
|
||||
|
||||
Công thức khớp client:
|
||||
- RBJ Audio-EQ-Cookbook biquad (eqproBiquadMagDb)
|
||||
- Imager 4-band M/S crossfeed g1=(w+100)/200, g2=(100-w)/200
|
||||
- Maximizer: boost -> atan soft-clip -> upward comp (dry + gain*comp) -> hard clip ceiling
|
||||
- Compressor: soft-knee DynamicsCompressor + one-pole attack/release envelope
|
||||
- Limiter: WaveShaper tanh k=1/tLin
|
||||
- Exciter: highpass 2k -> atan(k=3), dry 1.0, wet 0.6*drive/100
|
||||
- Rebalance: M/S L/R crossfeed a=(m+s)/2, b=(m-s)/2
|
||||
"""
|
||||
import numpy as np
|
||||
from scipy.signal import sosfilt
|
||||
|
||||
|
||||
def _clamp(v, lo, hi, default=0.0):
|
||||
"""Mirror client clamp(): missing/NaN -> default (neutral), khong bao gio NaN."""
|
||||
try:
|
||||
n = float(v)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
if not np.isfinite(n):
|
||||
return default
|
||||
return min(hi, max(lo, n))
|
||||
|
||||
|
||||
def _biquad(kind, f0, fs, gain_db=0.0, q=0.707):
|
||||
"""RBJ biquad (b, a) normalized a0=1 — dung cong thuc client eqproBiquadMagDb."""
|
||||
f0 = min(max(float(f0), 20.0), fs * 0.45)
|
||||
w0 = 2 * np.pi * f0 / fs
|
||||
cw = np.cos(w0)
|
||||
sw = np.sin(w0)
|
||||
alpha = sw / (2 * max(0.05, q))
|
||||
A = 10 ** (_clamp(gain_db, -24, 24) / 40)
|
||||
if kind == "peaking":
|
||||
b = [1 + alpha * A, -2 * cw, 1 - alpha * A]
|
||||
a = [1 + alpha / A, -2 * cw, 1 - alpha / A]
|
||||
elif kind == "lowshelf":
|
||||
b = [A * ((A + 1) - (A - 1) * cw + 2 * np.sqrt(A) * alpha),
|
||||
2 * A * ((A - 1) - (A + 1) * cw),
|
||||
A * ((A + 1) - (A - 1) * cw - 2 * np.sqrt(A) * alpha)]
|
||||
a = [(A + 1) + (A - 1) * cw + 2 * np.sqrt(A) * alpha,
|
||||
-2 * ((A - 1) + (A + 1) * cw),
|
||||
(A + 1) + (A - 1) * cw - 2 * np.sqrt(A) * alpha]
|
||||
elif kind == "highshelf":
|
||||
b = [A * ((A + 1) + (A - 1) * cw + 2 * np.sqrt(A) * alpha),
|
||||
-2 * A * ((A - 1) + (A + 1) * cw),
|
||||
A * ((A + 1) + (A - 1) * cw - 2 * np.sqrt(A) * alpha)]
|
||||
a = [(A + 1) - (A - 1) * cw + 2 * np.sqrt(A) * alpha,
|
||||
2 * ((A - 1) - (A + 1) * cw),
|
||||
(A + 1) - (A - 1) * cw - 2 * np.sqrt(A) * alpha]
|
||||
elif kind == "highpass":
|
||||
b = [(1 + cw) / 2, -(1 + cw), (1 + cw) / 2]
|
||||
a = [1 + alpha, -2 * cw, 1 - alpha]
|
||||
elif kind == "lowpass":
|
||||
b = [(1 - cw) / 2, 1 - cw, (1 - cw) / 2]
|
||||
a = [1 + alpha, -2 * cw, 1 - alpha]
|
||||
else:
|
||||
raise ValueError(f"unsupported biquad kind: {kind}")
|
||||
b = np.asarray(b, dtype=np.float64) / a[0]
|
||||
a = np.asarray(a, dtype=np.float64) / a[0]
|
||||
return np.concatenate([b, a])
|
||||
|
||||
|
||||
def _sosfilt_chain(x, sos_list):
|
||||
"""Ap day biquad noi tiep len (2, N) — mot lan sosfilt moi kenh."""
|
||||
if not sos_list:
|
||||
return np.array(x, dtype=np.float64, copy=True)
|
||||
sos = np.asarray(sos_list, dtype=np.float64)
|
||||
out = np.empty((2, x.shape[1]), dtype=np.float64)
|
||||
for ch in range(2):
|
||||
out[ch] = sosfilt(sos, np.asarray(x[ch], dtype=np.float64))
|
||||
return out
|
||||
|
||||
|
||||
def _compressor(x, threshold_db, ratio, knee, attack_s, release_s,
|
||||
makeup_db=0.0, fs=44100.0):
|
||||
"""Soft-knee feedforward compressor (gan dung DynamicsCompressor cua WebAudio)
|
||||
— stereo-linked (detect tren max |L|,|R|), one-pole attack/release."""
|
||||
x64 = np.asarray(x, dtype=np.float64)
|
||||
n = x64.shape[1]
|
||||
if n == 0:
|
||||
return x64
|
||||
level_db = 20 * np.log10(np.max(np.abs(x64), axis=0) + 1e-12)
|
||||
t = threshold_db
|
||||
k = knee
|
||||
gr_db = np.zeros(n, dtype=np.float64)
|
||||
above = level_db >= t + k / 2
|
||||
mid = (level_db > t - k / 2) & (level_db < t + k / 2)
|
||||
gr_db[above] = (1 - 1 / ratio) * (t - level_db[above])
|
||||
gr_db[mid] = (1 - 1 / ratio) * (level_db[mid] - t + k / 2) ** 2 / (2 * k)
|
||||
|
||||
a_att = np.exp(-1 / (attack_s * fs)) if attack_s > 0 else 0.0
|
||||
a_rel = np.exp(-1 / (release_s * fs)) if release_s > 0 else 0.0
|
||||
env = np.empty(n, dtype=np.float64)
|
||||
cur = 0.0
|
||||
prev = 0.0
|
||||
for i in range(n):
|
||||
g = gr_db[i]
|
||||
if g < prev:
|
||||
cur = a_att * cur + (1 - a_att) * g
|
||||
else:
|
||||
cur = a_rel * cur + (1 - a_rel) * g
|
||||
env[i] = cur
|
||||
prev = g
|
||||
gain_lin = 10 ** ((env + makeup_db) / 20)
|
||||
return x64 * gain_lin
|
||||
|
||||
|
||||
def _apply_eq(x, s, fs):
|
||||
eq_on = bool(s.get("eqActive"))
|
||||
sos = [
|
||||
_biquad("lowshelf", 100, fs, _clamp(s.get("eqLowGain"), -24, 24) if eq_on else 0),
|
||||
_biquad("peaking", 822, fs, _clamp(s.get("eqMid1Gain"), -24, 24) if eq_on else 0, q=0.7),
|
||||
_biquad("peaking", 3200, fs, _clamp(s.get("eqMid2Gain"), -24, 24) if eq_on else 0, q=1.2),
|
||||
_biquad("highshelf", 10000, fs, _clamp(s.get("eqHighGain"), -24, 24) if eq_on else 0),
|
||||
]
|
||||
return _sosfilt_chain(x, sos)
|
||||
|
||||
|
||||
def _apply_imager(x, s, fs):
|
||||
im_on = bool(s.get("imagerActive"))
|
||||
bands = [
|
||||
[("lowpass", 100)],
|
||||
[("highpass", 100), ("lowpass", 1000)],
|
||||
[("highpass", 1000), ("lowpass", 6000)],
|
||||
[("highpass", 6000)],
|
||||
]
|
||||
widths = [s.get(f"w{i}") for i in (1, 2, 3, 4)]
|
||||
out = np.zeros_like(x)
|
||||
for bf, w in zip(bands, widths):
|
||||
width = _clamp(w, 0, 200, 100) if im_on else 100
|
||||
g1 = (width + 100) / 200
|
||||
g2 = (100 - width) / 200
|
||||
y = _sosfilt_chain(x, [_biquad(k, f, fs) for k, f in bf])
|
||||
out[0] += g1 * y[0] + g2 * y[1]
|
||||
out[1] += g1 * y[1] + g2 * y[0]
|
||||
return out
|
||||
|
||||
|
||||
def _apply_maximizer(x, s):
|
||||
on = bool(s.get("maximizerActive"))
|
||||
boost = 10 ** (_clamp(s.get("maxGain"), -60, 30) / 20) if on else 1.0
|
||||
boosted = x * boost
|
||||
soft = _clamp(s.get("maxSoftClip"), 0, 100, 0)
|
||||
if on and soft > 0:
|
||||
k = 1 + (soft / 100) * 10
|
||||
dry = np.arctan(boosted * k) / np.arctan(k)
|
||||
else:
|
||||
dry = boosted
|
||||
up = _clamp(s.get("maxUpward"), 0, 30, 0)
|
||||
if on and up > 0:
|
||||
up_gain = 10 ** (up / 20) - 1.0
|
||||
comp = _compressor(boosted, threshold_db=-30, ratio=4.0, knee=10.0,
|
||||
attack_s=0.01, release_s=0.1, makeup_db=0.0)
|
||||
out = dry + up_gain * comp
|
||||
else:
|
||||
out = dry
|
||||
ceil_db = _clamp(s.get("ceiling"), -60, 0, -0.1) if on else -0.1
|
||||
c = 10 ** (ceil_db / 20)
|
||||
return np.clip(out, -c, c)
|
||||
|
||||
|
||||
def _apply_compressor(x, s):
|
||||
if not bool(s.get("compActive")):
|
||||
return x
|
||||
makeup = 10 ** (_clamp(s.get("compMakeup"), 0, 12) / 20)
|
||||
return _compressor(x, threshold_db=_clamp(s.get("compThreshold"), -60, 0),
|
||||
ratio=_clamp(s.get("compRatio"), 1, 20), knee=8.0,
|
||||
attack_s=0.02, release_s=0.25, makeup_db=20 * np.log10(makeup))
|
||||
|
||||
|
||||
def _apply_limiter(x, s):
|
||||
if not bool(s.get("limActive")):
|
||||
return x
|
||||
t_lin = 10 ** (_clamp(s.get("limThreshold"), -24, 0, -1) / 20)
|
||||
k = 1 / max(0.02, t_lin)
|
||||
tk = np.tanh(k)
|
||||
return np.tanh(x * k) / tk
|
||||
|
||||
|
||||
def _apply_exciter(x, s, fs):
|
||||
if not bool(s.get("excActive")):
|
||||
return x
|
||||
wet = (_clamp(s.get("excDrive"), 0, 100, 0) / 100) * 0.6
|
||||
if wet <= 0:
|
||||
return x
|
||||
hp = _sosfilt_chain(x, [_biquad("highpass", 2000, fs, q=0.7)])
|
||||
k = 3
|
||||
sh = np.arctan(hp * k) / np.arctan(k)
|
||||
return x + wet * sh
|
||||
|
||||
|
||||
def _apply_rebalance(x, s):
|
||||
if not bool(s.get("rebalActive")):
|
||||
return x
|
||||
mid = 10 ** (_clamp(s.get("rebalMid"), -24, 24, 0) / 20)
|
||||
side = 10 ** (_clamp(s.get("rebalSide"), -24, 24, 0) / 20)
|
||||
a = (mid + side) / 2
|
||||
b = (mid - side) / 2
|
||||
L, R = x[0], x[1]
|
||||
return np.stack([a * L + b * R, b * L + a * R])
|
||||
|
||||
|
||||
def _apply_eqpro(x, mod, fs):
|
||||
params = mod.get("params") or {}
|
||||
amount = _clamp(params.get("amount"), 0, 200, 100) / 100.0
|
||||
sos = []
|
||||
for b in params.get("bands") or []:
|
||||
kind = b.get("type") or "peaking"
|
||||
f0 = _clamp(b.get("freq"), 20, 20000, 1000)
|
||||
q = _clamp(b.get("q"), 0.1, 18, 1.0)
|
||||
gain = (b.get("gain") or 0) * amount if b.get("active") is not False else 0
|
||||
sos.append(_biquad(kind, f0, fs, gain, q))
|
||||
return _sosfilt_chain(x, sos)
|
||||
|
||||
|
||||
def apply_mastering(buffer, settings, sample_rate):
|
||||
"""Ap mastering chain (client format mastering_settings) len (2, N) audio.
|
||||
|
||||
buffer: (2, N) — L row 0. settings: dict hoac None. Tra (2, N) float64
|
||||
(copy) da xu li; neu gate tat / khong co module -> tra copy khong doi."""
|
||||
if buffer.ndim != 2 or buffer.shape[0] != 2:
|
||||
raise ValueError("buffer phai la (2, N) stereo")
|
||||
x = np.asarray(buffer, dtype=np.float64)
|
||||
if not settings:
|
||||
return x.copy()
|
||||
if not settings.get("masterConnected") or settings.get("isBypassed"):
|
||||
return x.copy()
|
||||
chain = settings.get("chain") or []
|
||||
mods = [m for m in chain if isinstance(m, dict) and m.get("active")]
|
||||
if not mods:
|
||||
return x.copy()
|
||||
fs = float(sample_rate) if sample_rate else 44100.0
|
||||
for m in mods:
|
||||
t = m.get("type")
|
||||
if t == "eq":
|
||||
x = _apply_eq(x, settings, fs)
|
||||
elif t == "imager":
|
||||
x = _apply_imager(x, settings, fs)
|
||||
elif t == "maximizer":
|
||||
x = _apply_maximizer(x, settings)
|
||||
elif t == "compressor":
|
||||
x = _apply_compressor(x, settings)
|
||||
elif t == "limiter":
|
||||
x = _apply_limiter(x, settings)
|
||||
elif t == "exciter":
|
||||
x = _apply_exciter(x, settings, fs)
|
||||
elif t == "rebalance":
|
||||
x = _apply_rebalance(x, settings)
|
||||
elif t == "eqpro":
|
||||
x = _apply_eqpro(x, m, fs)
|
||||
# type 'carla' = pass-through (VST FX ngoai) — mirror client, bo qua
|
||||
return x
|
||||
@@ -470,10 +470,25 @@ class PythonRenderEngine:
|
||||
_cache={},
|
||||
)
|
||||
|
||||
# Normalization to prevent clipping
|
||||
max_peak = np.max(np.abs(master_buffer))
|
||||
if max_peak > 1.0:
|
||||
master_buffer /= max_peak
|
||||
# Mastering chain — mirror client WebAudio (app.jsx applyMasteringSettings /
|
||||
# rebuildMasteringGraph). Truoc day mastering_settings bi BO QUA hoan toan:
|
||||
# export WAV khong qua EQ/Imager/Maximizer -> file khac am nghe live.
|
||||
# Gio ap dung dung chain (thu tu settings['chain']) khi mastering bat.
|
||||
mastering_settings = project_json.get("mastering_settings")
|
||||
mastered = bool(mastering_settings and mastering_settings.get("masterConnected")
|
||||
and not mastering_settings.get("isBypassed"))
|
||||
if mastered:
|
||||
# Import lazy (scipy.signal) — giam thoi gian khoi dong engine
|
||||
from app.core.mastering_engine import apply_mastering
|
||||
master_buffer = apply_mastering(master_buffer, mastering_settings,
|
||||
self.sample_rate)
|
||||
# Hard clip [-1,1] — khớp WAV encoder client (browser) sau mastering
|
||||
np.clip(master_buffer, -1.0, 1.0, out=master_buffer)
|
||||
else:
|
||||
# Normalization to prevent clipping
|
||||
max_peak = np.max(np.abs(master_buffer))
|
||||
if max_peak > 1.0:
|
||||
master_buffer /= max_peak
|
||||
|
||||
# Write final output file (bit_depth: 16/24/32 → WAV PCM subtype)
|
||||
subtype_map = {16: "PCM_16", 24: "PCM_24", 32: "PCM_32"}
|
||||
|
||||
+10
-1
@@ -198,7 +198,7 @@ const isSfTrackEngine = (se) => {
|
||||
};
|
||||
const isVstTrackEngine = (se) => !!se && String(se.type || '').indexOf('vst') !== -1;
|
||||
// Track dùng VSTi + Carla local → route Carla (native GUI, realtime).
|
||||
const shouldRouteCarla = (se) => !!(window.SonicCarlaMidi && window.SonicCarlaMidi.shouldRoutePlayback(se));
|
||||
const shouldRouteCarla = () => false; // live playback 100% client (SF2 autosampled WASM) — không route Carla
|
||||
|
||||
// ── Native soundfont preview (standalone) ─────────────────────────────────
|
||||
// Render 1 note bằng backend native FluidSynth (pyfluidsynth) → play WAV.
|
||||
@@ -15492,6 +15492,11 @@ const App = () => {
|
||||
setInstrumentDropdownTrackId(null);
|
||||
setInstrumentDropdownBtnRect(null);
|
||||
setSynthCategory(null);
|
||||
// VSTi live playback: pre-warm autosample (SF2 backend) — note đầu không bị
|
||||
// delay (SonicSF cũng ensure lazily trong _playNoteFluid nếu chưa có).
|
||||
if (window.SonicVstiAutosample && instrumentId && !isSfInstrument) {
|
||||
window.SonicVstiAutosample.ensure({ plugin_id: instrumentId, type: 'vst3' });
|
||||
}
|
||||
// Trigger FluidSynth load + program change when soundfont instrument selected
|
||||
if (window.SonicSF && window.SonicSF.selectInstrument && instrumentId && isSfInstrument) {
|
||||
const sfId = instrumentId.replace('sf_', '');
|
||||
@@ -16398,6 +16403,10 @@ const App = () => {
|
||||
return;
|
||||
}
|
||||
updateActiveTracks(prev => prev.map(t => t.id === trackId ? { ...t, synth_engine: { ...(t.synth_engine || {}), preset_id: presetId } } : t));
|
||||
// Autosample lại theo preset mới (key khác) — live playback dùng âm preset
|
||||
if (window.SonicVstiAutosample && se.plugin_id) {
|
||||
window.SonicVstiAutosample.ensure({ plugin_id: se.plugin_id, preset_id: presetId, type: 'vst3' });
|
||||
}
|
||||
setInstrumentDropdownTrackId(null);
|
||||
setInstrumentDropdownBtnRect(null);
|
||||
var p = ((window.SonicRuntime && window.SonicRuntime.presets) || []).find(function (x) { return x.id === presetId; });
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -91,6 +91,8 @@ window.API_BASE_URL = window.API_BASE_URL || window.location.origin;
|
||||
// để UI preview / gán clip vào track / download
|
||||
midiRender: (payload) => apiRequest('/api/v1/plugins/midi-render', { method: 'POST', body: JSON.stringify(payload) }),
|
||||
soundfontRender: (payload) => apiRequest('/api/v1/plugins/soundfont-render', { method: 'POST', body: JSON.stringify(payload) }),
|
||||
// Autosample VSTi → SF2 (backend pedalboard) — live playback client-side qua WASM
|
||||
autosampleVsti: (payload) => apiRequest('/api/v1/plugins/autosample', { method: 'POST', body: JSON.stringify(payload) }),
|
||||
// Phát dãy MIDI notes realtime qua Carla bridge (OSC) — preview khi
|
||||
// pedalboard không render được plugin (VD VST2)
|
||||
carlaPlayNotes: (payload) => apiRequest('/api/v1/plugins/carla-play-notes', { method: 'POST', body: JSON.stringify(payload) }),
|
||||
|
||||
@@ -74,6 +74,10 @@ window.SonicRuntime = window.SonicRuntime || { loaded: false, capabilities: null
|
||||
window.SonicCarlaMidi = window.SonicCarlaMidi || {
|
||||
shouldRoute: function (synthEngine, isArmed) {
|
||||
try {
|
||||
// Live playback qua Carla TẮT mặc định: VSTi phát client-side qua SF2
|
||||
// autosampled (FluidSynth WASM) → masterBus → mastering → main out.
|
||||
// Bật lại = window.__enableCarlaLivePlayback = true (debug/legacy).
|
||||
if (window.__enableCarlaLivePlayback !== true) return false;
|
||||
var c = window.SonicRuntime && window.SonicRuntime.capabilities;
|
||||
if (!c || !c.features || !c.features.carla_local) return false;
|
||||
if (!isArmed) return false;
|
||||
@@ -85,6 +89,8 @@ window.SonicCarlaMidi = window.SonicCarlaMidi || {
|
||||
// user đã chủ động bấm Play trên item đó).
|
||||
shouldRoutePlayback: function (synthEngine) {
|
||||
try {
|
||||
// (xem shouldRoute) — live playback qua Carla tắt mặc định
|
||||
if (window.__enableCarlaLivePlayback !== true) return false;
|
||||
var c = window.SonicRuntime && window.SonicRuntime.capabilities;
|
||||
if (!c || !c.features || !c.features.carla_local) return false;
|
||||
var se = synthEngine || {};
|
||||
|
||||
@@ -464,7 +464,7 @@
|
||||
bank = bank !== undefined ? bank : (synthEngine.soundfont_bank || 0);
|
||||
program = program !== undefined ? program : (synthEngine.soundfont_program || 0);
|
||||
}
|
||||
var sfId = synthEngine && synthEngine.soundfont_id;
|
||||
var sfId = synthEngine && (synthEngine.soundfont_id || synthEngine.autosampled_sf_id);
|
||||
var engKey = (sfId || '') + ':' + bank + ':' + program;
|
||||
if (!_engineChMap[engKey]) {
|
||||
var channel = this.allocateChannel(bank);
|
||||
@@ -587,7 +587,36 @@
|
||||
if (isNaN(finalBank) || !isFinite(finalBank)) finalBank = 0;
|
||||
var finalProg = parseInt(usedProg);
|
||||
if (isNaN(finalProg) || !isFinite(finalProg)) finalProg = 0;
|
||||
var finalSfId = synthEngine ? synthEngine.soundfont_id : undefined;
|
||||
var finalSfId = synthEngine ? (synthEngine.soundfont_id || synthEngine.autosampled_sf_id) : undefined;
|
||||
// VSTi live playback (không Carla): chưa có SF2 autosampled → chờ
|
||||
// SonicVstiAutosample.ensure (1 request backend / plugin+preset) rồi
|
||||
// play qua FluidSynth WASM như soundfont thường — âm đi qua masterBus
|
||||
// → mastering → main out. ensure dedup in-flight + cooldown lỗi 60s.
|
||||
if (!finalSfId && window.SonicVstiAutosample && synthEngine && !synthEngine.soundfont_id
|
||||
&& String(synthEngine.type || '').indexOf('vst') !== -1 && !!synthEngine.plugin_id) {
|
||||
var _pendKey = ch + ':' + midiPitch;
|
||||
var _gen = _noteGeneration;
|
||||
_pendingNoteOns[_pendKey] = (_pendingNoteOns[_pendKey] || 0) + 1;
|
||||
window.SonicVstiAutosample.ensure(synthEngine).then(function (sfId) {
|
||||
if (_gen !== _noteGeneration) return;
|
||||
var _pend = _pendingNoteOns[_pendKey];
|
||||
if (_pend) {
|
||||
_pendingNoteOns[_pendKey] = _pend - 1;
|
||||
if (_pendingNoteOns[_pendKey] <= 0) delete _pendingNoteOns[_pendKey];
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
if (!sfId) {
|
||||
// Không autosample được (404/501) → fallback oscillator để note
|
||||
// KHÔNG câm; cooldown trong ensure chặn re-request.
|
||||
try { self._playNoteFallback(note, velocity, durationMs, startTime, program, null, channel, synthEngine); } catch (e) {}
|
||||
return;
|
||||
}
|
||||
synthEngine.autosampled_sf_id = sfId;
|
||||
doNote();
|
||||
});
|
||||
return;
|
||||
}
|
||||
var cachedCh = _channels[ch];
|
||||
// The note's own synth engine (track instrument) is
|
||||
// authoritative. Channel state is only a cache: it must never
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
// SonicForge VSTi Autosample Service
|
||||
// Live playback 100% client-side (không Carla): VSTi note → SF2 autosampled
|
||||
// (backend pedalboard, render 1 lần / plugin+preset) → FluidSynth WASM qua
|
||||
// masterBus → mastering → main out. ensure() dedup in-flight request + cooldown
|
||||
// lỗi (404/501/no pedalboard) để không spam server.
|
||||
window.SonicVstiAutosample = window.SonicVstiAutosample || {};
|
||||
|
||||
(function () {
|
||||
var LS_MAP_KEY = 'sf_vsti_autosample_map';
|
||||
var LS_FAIL_KEY = 'sf_vsti_autosample_fail';
|
||||
var FAIL_COOLDOWN_MS = 60000;
|
||||
|
||||
function readMap() {
|
||||
try { return JSON.parse(localStorage.getItem(LS_MAP_KEY) || '{}') || {}; } catch (e) { return {}; }
|
||||
}
|
||||
function writeMap(map) {
|
||||
try { localStorage.setItem(LS_MAP_KEY, JSON.stringify(map)); } catch (e) {}
|
||||
}
|
||||
function readFails() {
|
||||
try { return JSON.parse(localStorage.getItem(LS_FAIL_KEY) || '{}') || {}; } catch (e) { return {}; }
|
||||
}
|
||||
|
||||
// Key = plugin + preset → autosample lại khi đổi preset (âm preset mới).
|
||||
function keyFor(synthEngine) {
|
||||
if (!synthEngine) return '';
|
||||
return String(synthEngine.plugin_id || '') + '|' + String(synthEngine.preset_id || synthEngine.presetId || '');
|
||||
}
|
||||
|
||||
// SF id đã autosample cho engine (cache localStorage) — đồng bộ, không request.
|
||||
function sfIdFor(synthEngine) {
|
||||
var k = keyFor(synthEngine);
|
||||
if (!k) return null;
|
||||
var map = readMap();
|
||||
return (map[k] && map[k].sf_id) ? map[k].sf_id : null;
|
||||
}
|
||||
|
||||
function entryFor(synthEngine) {
|
||||
var k = keyFor(synthEngine);
|
||||
if (!k) return null;
|
||||
return readMap()[k] || null;
|
||||
}
|
||||
|
||||
var _inFlight = {}; // key → Promise<sf_id|null> — nhiều note cùng lúc chỉ 1 request
|
||||
|
||||
// Đảm bảo SF2 autosampled tồn tại. Trả Promise<sf_id|null>; null =
|
||||
// không autosample được → caller fallback (oscillator / không phát).
|
||||
function ensure(synthEngine) {
|
||||
var k = keyFor(synthEngine);
|
||||
if (!k) return Promise.resolve(null);
|
||||
var cached = sfIdFor(synthEngine);
|
||||
if (cached) return Promise.resolve(cached);
|
||||
if (_inFlight[k]) return _inFlight[k];
|
||||
var fails = readFails();
|
||||
if (fails[k] && (Date.now() - fails[k]) < FAIL_COOLDOWN_MS) {
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
if (!window.SonicAPI || !window.SonicAPI.autosampleVsti) {
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
_inFlight[k] = new Promise(function (resolve) {
|
||||
var payload = {
|
||||
instrument_id: synthEngine.plugin_id,
|
||||
preset_id: synthEngine.preset_id || synthEngine.presetId || null,
|
||||
preset_path: synthEngine.preset_path || null,
|
||||
preset_data: synthEngine.preset_data || null
|
||||
};
|
||||
window.SonicAPI.autosampleVsti(payload).then(function (res) {
|
||||
delete _inFlight[k];
|
||||
if (res && res.success && res.sf_id) {
|
||||
var map = readMap();
|
||||
map[k] = { sf_id: res.sf_id, size_bytes: res.size_bytes || 0, note_count: res.note_count || 0, plugin_id: synthEngine.plugin_id, preset_id: synthEngine.preset_id || synthEngine.presetId || '', ts: Date.now() };
|
||||
writeMap(map);
|
||||
resolve(res.sf_id);
|
||||
} else {
|
||||
resolve(null);
|
||||
}
|
||||
}).catch(function () {
|
||||
delete _inFlight[k];
|
||||
var fails2 = readFails();
|
||||
fails2[k] = Date.now();
|
||||
try { localStorage.setItem(LS_FAIL_KEY, JSON.stringify(fails2)); } catch (e) {}
|
||||
resolve(null);
|
||||
});
|
||||
});
|
||||
return _inFlight[k];
|
||||
}
|
||||
|
||||
// Xoá cache + cooldown — cho phép re-sample (đổi preset/plugin hoặc debug).
|
||||
function clearCache() {
|
||||
try { localStorage.removeItem(LS_MAP_KEY); } catch (e) {}
|
||||
try { localStorage.removeItem(LS_FAIL_KEY); } catch (e) {}
|
||||
}
|
||||
|
||||
window.SonicVstiAutosample = {
|
||||
keyFor: keyFor,
|
||||
sfIdFor: sfIdFor,
|
||||
entryFor: entryFor,
|
||||
ensure: ensure,
|
||||
clearCache: clearCache
|
||||
};
|
||||
})();
|
||||
@@ -33,12 +33,13 @@
|
||||
<script src="/static/vendor/react.production.min.js"></script>
|
||||
<script src="/static/vendor/react-dom.production.min.js"></script>
|
||||
<script src="/static/js/services/fluidsynthLoader.js?v=202607271245"></script>
|
||||
<script src="/static/js/services/runtime.js?v=202608101800"></script>
|
||||
<script src="/static/js/services/api.js?v=202608101800"></script>
|
||||
<script src="/static/js/services/runtime.js?v=202608111230"></script>
|
||||
<script src="/static/js/services/api.js?v=202608111230"></script>
|
||||
<script src="/static/js/services/vstiAutosample.js?v=202608111230"></script>
|
||||
<script src="/static/js/services/audioEngine.js?v=202607271016"></script>
|
||||
<script src="/static/js/services/storage.js?v=202608038200"></script>
|
||||
<script src="/static/js/services/soundfontStorage.js?v=202607271016"></script>
|
||||
<script src="/static/js/services/soundfontPlayer.js?v=202608101800"></script>
|
||||
<script src="/static/js/services/soundfontPlayer.js?v=202608111230"></script>
|
||||
<script src="/static/js/services/aiGateway.js?v=202608037200"></script>
|
||||
<script src="/static/js/services/dawCommandDispatcher.js?v=202607271016"></script>
|
||||
<script src="/static/js/services/pianoRollTabService.js?v=202607272044"></script>
|
||||
@@ -46,7 +47,7 @@
|
||||
<script src="/static/js/services/midiExtractor.js?v=202607281052"></script>
|
||||
<script src="/static/js/services/promptTemplateManager.js?v=202607281039"></script>
|
||||
<script src="/static/js/services/undoRedoEngine.js?v=202607290941"></script>
|
||||
<script src="/static/js/app.precompiled.js?v=202608102202" defer></script>
|
||||
<script src="/static/js/app.precompiled.js?v=202608111230" defer></script>
|
||||
<link rel="stylesheet" href="/static/css/styles.css?v=202607271016">
|
||||
<style>
|
||||
:root {
|
||||
|
||||
@@ -32,3 +32,85 @@ def test_render_project_bit_depth(tmp_path):
|
||||
out = str(tmp_path / f"render_{bd}.wav")
|
||||
engine.render_project(project, out, bit_depth=bd)
|
||||
assert sf.info(out).subtype == subtype
|
||||
|
||||
def _default_mastering():
|
||||
return {
|
||||
"masterConnected": True, "isBypassed": False,
|
||||
"eqActive": True, "eqLowGain": 1.5, "eqMid1Gain": -1.0, "eqMid2Gain": 2.0, "eqHighGain": 1.8,
|
||||
"imagerActive": True, "w1": 0, "w2": 115, "w3": 135, "w4": 150,
|
||||
"maximizerActive": True, "maxGain": 5.4, "maxUpward": 2.0, "maxSoftClip": 15, "ceiling": -0.1,
|
||||
"compActive": False, "compThreshold": -16, "compRatio": 3, "compMakeup": 2,
|
||||
"limActive": False, "limThreshold": -1.0,
|
||||
"excActive": False, "excDrive": 30,
|
||||
"rebalActive": False, "rebalMid": 0, "rebalSide": 0,
|
||||
"chain": [
|
||||
{"id": "mod_eq", "type": "eq", "active": True},
|
||||
{"id": "mod_imager", "type": "imager", "active": True},
|
||||
{"id": "mod_maximizer", "type": "maximizer", "active": True},
|
||||
],
|
||||
}
|
||||
|
||||
def test_apply_mastering_engine_gates_and_ceiling():
|
||||
"""apply_mastering: gate masterConnected/isBypassed/chain rỗng → copy;
|
||||
chain mặc định → khác input, peak ≤ ceiling (hard clip của Maximizer)."""
|
||||
import numpy as np
|
||||
from app.core.mastering_engine import apply_mastering
|
||||
sr = 44100
|
||||
t = np.arange(sr // 5) / sr
|
||||
buf = np.stack([np.sin(2 * np.pi * 220 * t) * 0.9,
|
||||
np.sin(2 * np.pi * 220 * t + 0.5) * 0.8]).astype(np.float32)
|
||||
s = _default_mastering()
|
||||
assert np.array_equal(apply_mastering(buf, None, sr), buf)
|
||||
assert np.array_equal(apply_mastering(buf, {**s, "masterConnected": False}, sr), buf)
|
||||
assert np.array_equal(apply_mastering(buf, {**s, "isBypassed": True}, sr), buf)
|
||||
assert np.array_equal(apply_mastering(buf, {**s, "chain": []}, sr), buf)
|
||||
out = apply_mastering(buf, s, sr)
|
||||
assert out.shape == buf.shape and not np.array_equal(out, buf)
|
||||
assert np.max(np.abs(out)) <= 10 ** (-0.1 / 20) + 1e-6 # ceiling -0.1 dB
|
||||
|
||||
def test_render_project_mastering(tmp_path):
|
||||
"""render_project: có mastering_settings → WAV khác bản không mastering,
|
||||
peak ≤ 1.0 (hard clip sau mastering như WAV encoder client)."""
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from app.config import settings
|
||||
engine = PythonRenderEngine()
|
||||
# 1s 440Hz tone làm nguồn audio thật (project rỗng = silence → mastering
|
||||
# của silence = silence, không test được gì)
|
||||
sr = engine.sample_rate
|
||||
t = np.arange(sr) / sr
|
||||
tone = (0.9 * np.sin(2 * np.pi * 440 * t)).astype(np.float32)
|
||||
src_path = os.path.join(settings.UPLOADS_DIR, "mastering_test_tone.wav")
|
||||
sf.write(src_path, tone, sr)
|
||||
try:
|
||||
project = {
|
||||
"metadata": {"bpm": 120, "time_signature_numerator": 4},
|
||||
"main_session": {
|
||||
"length_bars": 4.0,
|
||||
"tracks": [{
|
||||
"type": "AUDIO", "volume_db": 0.0, "pan": 0.0, "mute": False,
|
||||
"items": [{
|
||||
"type": "AUDIO_ITEM", "start_bar": 0.0, "duration_bars": 4.0,
|
||||
"clip_start_offset_bars": 0.0,
|
||||
"source_data": {"audio_file_url": "/static/audio/uploads/mastering_test_tone.wav", "gain": 1.0},
|
||||
}],
|
||||
}],
|
||||
},
|
||||
"section_store": {},
|
||||
}
|
||||
out_plain = str(tmp_path / "plain.wav")
|
||||
engine.render_project(project, out_plain, bit_depth=16)
|
||||
plain, _ = sf.read(out_plain, dtype="float32")
|
||||
assert np.max(np.abs(plain)) > 0.01
|
||||
|
||||
out_mast = str(tmp_path / "mastered.wav")
|
||||
engine.render_project({**project, "mastering_settings": _default_mastering()},
|
||||
out_mast, bit_depth=16)
|
||||
mast, _ = sf.read(out_mast, dtype="float32")
|
||||
assert not np.array_equal(mast, plain)
|
||||
assert np.max(np.abs(mast)) <= 1.0 + 1e-6
|
||||
finally:
|
||||
try:
|
||||
os.remove(src_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
+53
-24
@@ -118,6 +118,49 @@ def _render_note(vst, note, sr, dur, release, velocity):
|
||||
return np.clip(audio * 32767, -32768, 32767).astype(np.int16)
|
||||
|
||||
|
||||
def autosample_sf2(instrument_id, out_path, preset_id=None, preset_path=None,
|
||||
preset_data_b64=None, low=36, high=96, step=2, duration=2.5,
|
||||
release=1.0, velocity=100, sample_rate=44100, name=None, log=None):
|
||||
"""Auto-sample VSTi -> SF2 (16-bit mono). Dung cho server endpoint
|
||||
/api/v1/plugins/autosample va CLI main().
|
||||
|
||||
Returns dict {note_count, out_path, size_bytes}.
|
||||
Raises FileNotFoundError neu plugin khong tim thay, RuntimeError neu
|
||||
pedalboard khong kha dung hoac khong render duoc not nao.
|
||||
"""
|
||||
from app.core.vst_engine import PluginManager, apply_preset_to_plugin, HAS_PEDALBOARD
|
||||
if not HAS_PEDALBOARD:
|
||||
raise RuntimeError("pedalboard khong kha dung - khong auto-sample duoc")
|
||||
|
||||
vst = PluginManager().load_vst(instrument_id)
|
||||
if vst is None:
|
||||
raise FileNotFoundError(f"Khong tim thay VSTi: {instrument_id} - hay Scan trong Plugin Manager truoc")
|
||||
if preset_id or preset_path or preset_data_b64:
|
||||
apply_preset_to_plugin(vst, preset_id=preset_id, preset_path=preset_path,
|
||||
preset_data_b64=preset_data_b64)
|
||||
|
||||
def _log(msg):
|
||||
if log:
|
||||
log(msg)
|
||||
|
||||
samples = []
|
||||
for note in range(low, high + 1, step):
|
||||
frames = _render_note(vst, note, sample_rate, duration, release, velocity)
|
||||
if frames is None:
|
||||
_log(f"note {note}: silent, bo qua")
|
||||
continue
|
||||
samples.append({"note": note, "frames": frames})
|
||||
_log(f"note {note}: {frames.shape[0] / sample_rate:.2f}s")
|
||||
if not samples:
|
||||
raise RuntimeError("Khong render duoc not nao (plugin silent?)")
|
||||
|
||||
if name is None:
|
||||
name = os.path.basename(instrument_id)
|
||||
write_sf2(out_path, samples, sample_rate, name=name)
|
||||
size = os.path.getsize(out_path)
|
||||
_log(f"wrote {out_path} ({size // 1024} KB, {len(samples)} not)")
|
||||
return {"note_count": len(samples), "out_path": out_path, "size_bytes": size}
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
ap.add_argument("--instrument", required=True, help="plugin_id da scan (Plugin Manager)")
|
||||
@@ -133,35 +176,21 @@ def main():
|
||||
ap.add_argument("--sf3", action="store_true", help="convert SF2 -> SF3 sau khi sample")
|
||||
args = ap.parse_args()
|
||||
|
||||
from app.core.vst_engine import PluginManager, apply_preset_to_plugin, HAS_PEDALBOARD
|
||||
if not HAS_PEDALBOARD:
|
||||
sys.exit("pedalboard khong kha dung - khong auto-sample duoc")
|
||||
|
||||
vst = PluginManager().load_vst(args.instrument)
|
||||
if vst is None:
|
||||
sys.exit(f"Khong tim thay VSTi: {args.instrument} - hay Scan trong Plugin Manager truoc")
|
||||
if args.preset:
|
||||
apply_preset_to_plugin(vst, preset_path=args.preset)
|
||||
|
||||
samples = []
|
||||
for note in range(args.low, args.high + 1, args.step):
|
||||
frames = _render_note(vst, note, args.sample_rate, args.duration, args.release, args.velocity)
|
||||
if frames is None:
|
||||
print(f"note {note}: silent, bo qua")
|
||||
continue
|
||||
samples.append({"note": note, "frames": frames})
|
||||
print(f"note {note}: {frames.shape[0] / args.sample_rate:.2f}s")
|
||||
if not samples:
|
||||
sys.exit("Khong render duoc not nao (plugin silent?)")
|
||||
|
||||
write_sf2(args.out, samples, args.sample_rate, name=os.path.basename(args.instrument))
|
||||
print(f"wrote {args.out} ({os.path.getsize(args.out) // 1024} KB, {len(samples)} not)")
|
||||
try:
|
||||
result = autosample_sf2(
|
||||
args.instrument, args.out, preset_path=args.preset,
|
||||
low=args.low, high=args.high, step=args.step,
|
||||
duration=args.duration, release=args.release,
|
||||
velocity=args.velocity, sample_rate=args.sample_rate,
|
||||
)
|
||||
except (FileNotFoundError, RuntimeError) as e:
|
||||
sys.exit(str(e))
|
||||
print(f"wrote {result['out_path']} ({result['size_bytes'] // 1024} KB, {result['note_count']} not)")
|
||||
|
||||
if args.sf3:
|
||||
from app.core.soundfont_converter import SoundFontConverter
|
||||
p = SoundFontConverter().convert_sf2_to_sf3(args.out)
|
||||
print("sf3:", p)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user