T16: autosample VST2 via native bridge + SF2/SF3 export hoàn chỉnh, UI chip size/note_count, re-sample on preset change, tests

2026-08-11 18:04:34 +07:00
parent 0af834a8fa
commit 371b1ca665
8 changed files with 184 additions and 50 deletions
@@ -1144,17 +1144,22 @@ async def autosample_vsti(req: AutosampleRequest, current_user: dict = Depends(g
    Tra sf_id (bare uuid) de client tai SF2 va phat qua SonicSF nhu soundfont
    thuong. Chi phi autosample mot lan -> cache o client (localStorage)."""
    enforce_password_changed(current_user)
-    if not HAS_PEDALBOARD:
-        raise HTTPException(status_code=501, detail="pedalboard khong kha dung tren may nay")
+    _dirs = _effective_dirs().get("plugin_dirs") or []

    # tools/ nam ngoai package app - import voi fallback path
    try:
-        from tools.autosample_vsti import autosample_sf2 as _autosample_sf2
+        from tools.autosample_vsti import autosample_sf2 as _autosample_sf2, _is_vst2_path
    except ImportError:
        _root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
        if _root not in sys.path:
            sys.path.insert(0, _root)
-        from tools.autosample_vsti import autosample_sf2 as _autosample_sf2
+        from tools.autosample_vsti import autosample_sf2 as _autosample_sf2, _is_vst2_path
+
+    if not HAS_PEDALBOARD:
+        # VST2 native bridge khong can pedalboard — chi chan khi plugin la VST3
+        _p = PluginManager(extra_vst_dirs=_dirs)._scan_plugins().get(req.instrument_id)
+        if not _p or not _is_vst2_path(_p):
+            raise HTTPException(status_code=501, detail="pedalboard khong kha dung tren may nay")

    file_uuid = str(uuid.uuid4())
    dest_path = os.path.join(UPLOAD_SF_DIR, file_uuid + ".sf2")
@@ -1174,6 +1179,7 @@ async def autosample_vsti(req: AutosampleRequest, current_user: dict = Depends(g
            duration=req.duration, release=req.release,
            velocity=req.velocity, sample_rate=req.sample_rate,
            log=_log,
+            extra_vst_dirs=_dirs,
        )
    except FileNotFoundError as e:
        raise HTTPException(status_code=404, detail=str(e))
@@ -445,8 +445,10 @@ class NativeAudioService:

    def render_vst2_offline(self, plugin_path, notes, bpm=120.0, sr=44100,
                            gain_db=0.0, pan=0.0, lim_active=False,
-                            threshold_db=-1.0, master_gain_db=0.0):
-        """Offline VST2: SF_VST2_Process (raw) + mixer_math (Python mirror)."""
+                            threshold_db=-1.0, master_gain_db=0.0, tail_sec=0.5):
+        """Offline VST2: SF_VST2_Process (raw) + mixer_math (Python mirror).
+
+        tail_sec: duoi im lang sau note cuoi (autosample muon release tail)."""
        with _LOCK:
            dll = self._load_vst2_dll()
            ctx = _Vst2Ctx(dll)
@@ -466,7 +468,7 @@ class NativeAudioService:
                    events.append((int((start_s + dur_s) * sr), "off", pitch, 0))
                    total = max(total, start_s + dur_s)
                events.sort(key=lambda e: e[0])
-                n = int((max(total, 0.25) + 0.5) * sr)
+                n = int((max(total, 0.25) + float(tail_sec)) * sr)
                out = np.zeros((2, n), dtype=np.float32)
                block = 256
                pos = 0
@@ -5,6 +5,27 @@ import logging
 import wave

 logger = logging.getLogger(__name__)
+def _ensure_fluidsynth_runtime() -> None:
+    """Dua thu muc chua libfluidsynth DLL vao PATH de pyfluidsynth import duoc
+    (find_library do theo PATH tren Windows). Khong lam gi tren non-Windows."""
+    if os.name != "nt":
+        return
+    root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    candidates = [
+        os.path.join(root, "native_host", "build", "Release", "fluidsynth_runtime"),
+        os.path.join(root, "native_host", "fluidsynth_runtime"),
+    ]
+    path = os.environ.get("PATH", "")
+    parts = [os.path.normcase(p) for p in path.split(os.pathsep)]
+    for d in candidates:
+        if os.path.isdir(d) and any(
+            f.lower().startswith("libfluidsynth") and f.lower().endswith(".dll")
+            for f in os.listdir(d)
+        ):
+            if os.path.normcase(d) not in parts:
+                os.environ["PATH"] = d + os.pathsep + path
+            return
+

 SF_TARGET_DIRS = [
    "/opt/daw_engine/soundfonts",
@@ -193,8 +214,8 @@ class SoundFontConverter:
                # OGG loop pointers are relative to the individual decompressed sample
                new_sloop = (startloop - start) if (startloop > start and startloop <= end) else 0
                new_eloop = (endloop - start) if (endloop > start and endloop <= end) else 0
-                # Mark the sample as Ogg Vorbis compressed (FLUID_SAMPLETYPE_OGG_VORBIS = 0x20)
-                new_stype = sampletype | 0x20
+                # Mark the sample as Ogg Vorbis compressed (FLUID_SAMPLETYPE_OGG_VORBIS = 0x10)
+                new_stype = sampletype | 0x10
                new_shdr += data[base:base + 20]                       # sample name
                new_shdr += struct.pack("<IIIIi", new_start, new_end, new_sloop, new_eloop, rate)
                new_shdr += data[base + 40:base + 44]                  # originalpitch, correction, samplelink
@@ -282,35 +303,28 @@ class SoundFontConverter:
        """Verify a SoundFont actually loads and renders audible audio (guards
        against shipping malformed SF3 files that silently play nothing).

-        Uses the low-level CFFI binding (new_fluid_synth / write_float) — the
-        high-level Synth() class does not exist in this binding, so it is never
-        used here.
+        Uses the high-level Synth() binding (sfload/get_samples).
        """
        if not os.path.exists(path):
            return False
+        _ensure_fluidsynth_runtime()
        try:
            import fluidsynth as _fs
            import numpy as np
-            _settings = _fs.new_fluid_settings()
-            _fl = _fs.new_fluid_synth(_settings)
+            _synth = _fs.Synth()
            try:
-                h = _fs.fluid_synth_sfload(_fl, path.encode("utf-8"), 1)
-                if h < 0:
+                fid = _synth.sfload(path)
+                if fid == -1:
                    return False
-                _fs.fluid_synth_program_select(_fl, 0, h, 0, 0)
-                _fs.fluid_synth_noteon(_fl, 0, 60, 100)
-                frames = 8820  # 0.2s
-                buf = np.zeros(frames * 2, dtype=np.float32)
-                _fs.fluid_synth_write_float(
-                    _fl, frames, buf.ctypes.data, 0, 1,
-                    buf.ctypes.data + frames * 4, 0, 1
-                )
-                _fs.fluid_synth_noteoff(_fl, 0, 60)
-                rms = float(np.sqrt(np.mean(buf ** 2)))
+                _synth.program_select(0, fid, 0, 0)
+                _synth.noteon(0, 60, 100)
+                buf = _synth.get_samples(8820)  # int16 stereo, 0.2s
+                _synth.noteoff(0, 60)
+                rms = float(np.sqrt(np.mean((buf.astype(np.float32) / 32768.0) ** 2)))
                return rms > 1e-4
            finally:
                try:
-                    _fs.delete_fluid_synth(_fl)
+                    _synth.delete()
                except Exception:
                    pass
        except Exception:
@@ -453,7 +467,7 @@ class SoundFontConverter:
                new_loopstart = loopstart + new_start if (loopstart or loopend) else 0
                new_loopend = loopend + new_start if (loopstart or loopend) else 0
                # Clear the Ogg Vorbis flag; keep mono/left/right/linked flags
-                new_stype = sampletype & ~0x20
+                new_stype = sampletype & ~0x10
                new_shdr += name
                new_shdr += struct.pack("<IIIIi", new_start, new_end, new_loopstart, new_loopend, rate)
                new_shdr += data[base + 40:base + 44]
@@ -7823,6 +7823,12 @@ const PianoRollTabEditor = ({ st, zoom, bpm, viewportWidth, activeTracks, onClos
  React.useEffect(() => {
    try { localStorage.setItem('sf_pr_zoom', String(rollZoom)); } catch (e) { }
  }, [rollZoom]);
+  // Autosample VSTi xong → re-render chip size/note_count (subtab header)
+  React.useEffect(function () {
+    const _h = () => setRenderTick(t => t + 1);
+    window.addEventListener('sf:autosample-update', _h);
+    return () => window.removeEventListener('sf:autosample-update', _h);
+  }, []);
  const [aiBarStart, setAiBarStart] = React.useState(0);
  const [aiBarEnd, setAiBarEnd] = React.useState(4);

@@ -9915,7 +9921,10 @@ const beatSec = 60.0 / (parseInt(bpm) || 120);
      onClick: () => onInstrumentSelect && onInstrumentSelect(st.trackId),
      title: st.instrumentName || "Synth",
      className: `px-1.5 py-0.5 text-[10px] rounded font-mono font-bold border transition flex items-center gap-0.5 max-w-[70px] ${st.instrumentName ? 'bg-violet-900 text-violet-300 border-violet-700' : 'bg-zinc-800 text-zinc-500 border-transparent hover:text-zinc-300'}`
-    }, React.createElement("i", { "data-lucide": "music", className: "w-3 h-3 shrink-0" }), React.createElement("span", { className: "truncate text-[9px]" }, activeParentTrackName ? ('(' + activeParentTrackName + ') ' + (st.instrumentName || 'Synth')) : (st.instrumentName || 'Synth'))), React.createElement("div", {
+    }, React.createElement("i", { "data-lucide": "music", className: "w-3 h-3 shrink-0" }), React.createElement("span", { className: "truncate text-[9px]" }, activeParentTrackName ? ('(' + activeParentTrackName + ') ' + (st.instrumentName || 'Synth')) : (st.instrumentName || 'Synth'))), (function() {
+      var _e = (typeof window.SonicVstiAutosample !== 'undefined') ? window.SonicVstiAutosample.entryFor(st.synth_engine) : null;
+      return _e ? /*#__PURE__*/React.createElement("span", { className: "px-1 py-0.5 bg-zinc-900 border border-zinc-800 text-emerald-400/90 rounded text-[9px] font-mono whitespace-nowrap shrink-0", title: "Autosampled: " + (_e.note_count || 0) + " notes" }, window.SonicVstiAutosample.formatSize(_e.size_bytes) + " \u00b7 " + (_e.note_count || 0) + "n") : null;
+    })(), React.createElement("div", {
      className: "flex items-center gap-1 ml-1 text-xs"
    }, React.createElement("span", { className: "text-zinc-500" }, "AI:"), React.createElement("input", {
      type: "number", value: aiBarStart, onChange: e => setAiBarStart(parseInt(e.target.value) || 0),
@@ -15374,6 +15383,13 @@ const App = () => {
  };
  const [instrumentSelectorData, setInstrumentSelectorData] = useState(null);
  const [instrumentRefreshKey, setInstrumentRefreshKey] = useState(0);
+  // Autosample VSTi xong → re-render chip size/note_count (track strip)
+  const [autosampleVersion, setAutosampleVersion] = React.useState(0);
+  React.useEffect(() => {
+    const _h = () => setAutosampleVersion(v => v + 1);
+    window.addEventListener('sf:autosample-update', _h);
+    return () => window.removeEventListener('sf:autosample-update', _h);
+  }, []);
  const openInstrumentSelector = trackId => {
    setInstrumentSelectorTrackId(trackId);
    setSfPresetSearchQuery('');
@@ -29708,7 +29724,10 @@ STRICT CONSTRAINTS:
      }, /*#__PURE__*/React.createElement("i", {
        "data-lucide": "music",
        className: "w-3 h-3"
-      })), /*#__PURE__*/React.createElement("span", { className: "truncate text-[10px]" }, track.instrumentName || track.instrumentId || "Synth"), /*#__PURE__*/React.createElement("i", { "data-lucide": "chevron-down", className: "w-3 h-3 shrink-0" }))), /*#__PURE__*/React.createElement("div", {
+      })), /*#__PURE__*/React.createElement("span", { className: "truncate text-[10px]" }, track.instrumentName || track.instrumentId || "Synth"), /*#__PURE__*/React.createElement("i", { "data-lucide": "chevron-down", className: "w-3 h-3 shrink-0" }))), (function() {
+        var _e = (typeof window.SonicVstiAutosample !== 'undefined') ? window.SonicVstiAutosample.entryFor(track.synth_engine) : null;
+        return _e ? /*#__PURE__*/React.createElement("span", { className: "px-1.5 py-0.5 bg-zinc-900 border border-zinc-800 text-emerald-400/90 rounded text-[9px] font-mono whitespace-nowrap shrink-0", title: "Autosampled: " + (_e.note_count || 0) + " notes" }, window.SonicVstiAutosample.formatSize(_e.size_bytes) + " \u00b7 " + (_e.note_count || 0) + "n") : null;
+      })(), /*#__PURE__*/React.createElement("div", {
        onMouseDown: e => handleTrackResizeMouseDown(e, track.id),
        className: "absolute bottom-0 left-0 right-0 h-1 cursor-ns-resize z-30 hover:bg-cyan-500/50 transition-colors",
        onClick: e => e.stopPropagation()
@@ -27,6 +27,17 @@ window.SonicVstiAutosample = window.SonicVstiAutosample || {};
        } catch (e) {}
    }

+    function formatSize(bytes) {
+        if (!bytes) return '0 KB';
+        if (bytes >= 1048576) return (bytes / 1048576).toFixed(1) + ' MB';
+        return Math.max(1, Math.round(bytes / 1024)) + ' KB';
+    }
+
+    // UI re-render (chip hien thi note_count/size) khi autosample xong/fail
+    function notify(k) {
+        try { window.dispatchEvent(new CustomEvent('sf:autosample-update', { detail: { key: k } })); } catch (e) {}
+    }
+
    // Key = plugin + preset → autosample lại khi đổi preset (âm preset mới).
    function keyFor(synthEngine) {
        if (!synthEngine) return '';
@@ -65,6 +76,7 @@ window.SonicVstiAutosample = window.SonicVstiAutosample || {};
        }
        if (!window.SonicAPI || !window.SonicAPI.autosampleVsti) {
            recordFail(k, 'API autosample khong kha dung');
+            notify(k);
            return Promise.resolve(null);
        }
        _inFlight[k] = new Promise(function (resolve) {
@@ -80,14 +92,17 @@ window.SonicVstiAutosample = window.SonicVstiAutosample || {};
                    var map = readMap();
                    map[k] = { sf_id: res.sf_id, size_bytes: res.size_bytes || 0, note_count: res.note_count || 0, plugin_id: synthEngine.plugin_id, preset_id: synthEngine.preset_id || synthEngine.presetId || '', ts: Date.now() };
                    writeMap(map);
+                    notify(k);
                    resolve(res.sf_id);
                } else {
                    recordFail(k, (res && res.error) ? String(res.error) : 'Server khong tra sf_id');
+                    notify(k);
                    resolve(null);
                }
            }).catch(function (err) {
                delete _inFlight[k];
                recordFail(k, (err && err.message) ? String(err.message) : 'Loi mang / server autosample');
+                notify(k);
                resolve(null);
            });
        });
@@ -115,6 +130,7 @@ window.SonicVstiAutosample = window.SonicVstiAutosample || {};
        keyFor: keyFor,
        sfIdFor: sfIdFor,
        entryFor: entryFor,
+        formatSize: formatSize,
        ensure: ensure,
        failReasonFor: failReasonFor,
        clearCache: clearCache
@@ -1,11 +1,15 @@
 """Tests cho tools/autosample_vsti.py — SF2 writer tự-sinh phải là RIFF/sfbk
 hợp lệ mà sf2utils parse được (không cần VSTi/pedalboard)."""
 import os
+import sys
 import numpy as np
 import pytest

 from tools.autosample_vsti import write_sf2

+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+FAKE_VST2 = os.path.join(ROOT, "native_host", "tests", "fake_vst2.dll")
+
 sf2utils = pytest.importorskip("sf2utils.sf2parse")


@@ -53,3 +57,20 @@ def test_write_sf2_odd_name_no_corruption(tmp_path, caplog):
            sf2 = sf2utils.Sf2File(f)
    assert len([s for s in sf2.samples if s.end > s.start]) == 1
    assert not any("corrupted" in r.message for r in caplog.records)
+
+def test_autosample_vst2_native_bridge(tmp_path):
+    """Nhanh VST2: render qua native bridge (fake_vst2.dll) → SF2 hop le,
+    khong can pedalboard. Skip tren nen khong phai Windows / thieu DLL."""
+    if sys.platform.startswith("win") and os.path.isfile(FAKE_VST2):
+        from tools.autosample_vsti import autosample_sf2
+        out = str(tmp_path / "vst2.sf2")
+        res = autosample_sf2("fake_vst2", out, low=60, high=64, step=2,
+                             duration=0.3, release=0.2,
+                             extra_vst_dirs=[os.path.dirname(FAKE_VST2)])
+        assert res["note_count"] == 3
+        assert res["size_bytes"] == os.path.getsize(out)
+        with open(out, "rb") as f:
+            sf2 = sf2utils.Sf2File(f)
+        assert len([s for s in sf2.samples if s.end > s.start]) == 3
+    else:
+        pytest.skip("fake_vst2.dll chi chay Windows (bridge ctypes WinDLL)")
@@ -64,20 +64,20 @@ def write_sf2(out_path, samples, sample_rate=44100, name="AutoSampled"):
    phdr += name.encode("ascii", "replace")[:20].ljust(20, b"\x00")
    phdr += struct.pack("<HHHIII", 0, 0, 0, 0, 0, 0)   # preset=0 bank=0 bagNdx=0 libr genre morph
    phdr += b"EOP".ljust(20, b"\x00") + struct.pack("<HHHIII", 0, 0, 1, 0, 0, 0)
-    pbag = struct.pack("<HH", 0, 0) + struct.pack("<HH", 0, 0)
+    pbag = struct.pack("<HH", 0, 0) + struct.pack("<HH", 1, 0)
    pmod = b"\x00" * 10
-    pgen = struct.pack("<HH", 0, 0)
+    pgen = struct.pack("<HH", 41, 0) + struct.pack("<HH", 0, 0)
    inst = bytearray()
    inst += b"AutoSampled".ljust(20, b"\x00") + struct.pack("<H", 0)
-    inst += b"EOI".ljust(20, b"\x00") + struct.pack("<H", 1)
+    inst += b"EOI".ljust(20, b"\x00") + struct.pack("<H", n)  # bagNdx = chi so record terminal zone (ibag co n+1 records)
    ibag = b"".join(struct.pack("<HH", i * 3, 0) for i in range(n + 1))
    imod = b"\x00" * 10
    igen = bytearray()
    for i, s in enumerate(samples):
        note = int(s["note"]) & 0xFF
-        igen += struct.pack("<HH", 60, note | (note << 8))  # keyRange lo=hi=note
-        igen += struct.pack("<HH", 69, i)                    # sampleID
-        igen += struct.pack("<HH", 74, note)                 # overridingRootKey
+        igen += struct.pack("<HH", 43, note | (note << 8))  # keyRange lo=hi=note
+        igen += struct.pack("<HH", 58, note)                # overridingRootKey
+        igen += struct.pack("<HH", 53, i)                   # sampleID (phai la gen cuoi zone)
    igen += struct.pack("<HH", 0, 0)
    shdr = bytearray()
    for i, s in enumerate(samples):
@@ -118,34 +118,88 @@ def _render_note(vst, note, sr, dur, release, velocity):
    return np.clip(audio * 32767, -32768, 32767).astype(np.int16)


+def _postprocess(buf, sr):
+    """Stereo float32 (2,N) -> int16 mono, peak 0.9 (chung cho 2 engine)."""
+    mono = buf.mean(axis=0)
+    peak = float(np.max(np.abs(mono)))
+    if peak < 1e-5:
+        return None
+    above = np.nonzero(np.abs(mono) > peak * 0.001)[0]
+    start = int(above[0]) if above.size else 0
+    audio = mono[start:] / peak * 0.9
+    return np.clip(audio * 32767, -32768, 32767).astype(np.int16)
+
+
+def _is_vst2_path(path):
+    """VST2 = .dll/.so dung rieng (Windows VST3 la FOLDER .vst3, .so VST3
+    co duoi .vst3.so) — native bridge render duoc; pedalboard thi khong."""
+    low = (path or "").lower()
+    if ".vst3" in low:
+        return False
+    return low.endswith(".dll") or low.endswith(".so")
+
+
+def _render_note_native(plugin_path, note, sr, dur, release, velocity):
+    """VST2 qua native bridge (Windows) — pedalboard khong ho tro VST2."""
+    from app.core.native_audio_service import get_service
+    bpm = 120.0
+    y = get_service().render_vst2_offline(
+        plugin_path,
+        [{"start_beat": 0.0, "duration_beats": dur * bpm / 60.0,
+          "note": int(note) & 0x7F, "velocity": int(velocity)}],
+        bpm=bpm, sr=sr, tail_sec=release,
+    )
+    return _postprocess(y, sr)
+
+
 def autosample_sf2(instrument_id, out_path, preset_id=None, preset_path=None,
                   preset_data_b64=None, low=36, high=96, step=2, duration=2.5,
-                   release=1.0, velocity=100, sample_rate=44100, name=None, log=None):
+                   release=1.0, velocity=100, sample_rate=44100, name=None, log=None,
+                   extra_vst_dirs=None):
    """Auto-sample VSTi -> SF2 (16-bit mono). Dung cho server endpoint
    /api/v1/plugins/autosample va CLI main().

+    VST3 -> pedalboard (ap preset neu co); VST2 (.dll/.so khong .vst3) ->
+    native bridge (Windows) — pedalboard khong ho tro VST2, preset VST2 chua
+    qua bridge duoc nen bo qua.
+
    Returns dict {note_count, out_path, size_bytes}.
    Raises FileNotFoundError neu plugin khong tim thay, RuntimeError neu
-    pedalboard khong kha dung hoac khong render duoc not nao.
+    engine khong kha dung hoac khong render duoc not nao.
    """
    from app.core.vst_engine import PluginManager, apply_preset_to_plugin, HAS_PEDALBOARD
-    if not HAS_PEDALBOARD:
-        raise RuntimeError("pedalboard khong kha dung - khong auto-sample duoc")

-    vst = PluginManager().load_vst(instrument_id)
-    if vst is None:
+    plugin_path = PluginManager(extra_vst_dirs=extra_vst_dirs or [])._scan_plugins().get(instrument_id)
+    if not plugin_path:
        raise FileNotFoundError(f"Khong tim thay VSTi: {instrument_id} - hay Scan trong Plugin Manager truoc")
-    if preset_id or preset_path or preset_data_b64:
-        apply_preset_to_plugin(vst, preset_id=preset_id, preset_path=preset_path,
-                               preset_data_b64=preset_data_b64)

    def _log(msg):
        if log:
            log(msg)

+    if _is_vst2_path(plugin_path):
+        # VST2: native bridge (Windows) — pedalboard khong ho tro. Preset chua
+        # export qua bridge (VST2AudioEngine chua co chunk/preset) -> bo qua.
+        if preset_id or preset_path or preset_data_b64:
+            _log("VST2: native bridge chua ho tro preset - bo qua, sample preset mac dinh")
+        def _render(note):
+            return _render_note_native(plugin_path, note, sample_rate, duration,
+                                       release, velocity)
+    else:
+        if not HAS_PEDALBOARD:
+            raise RuntimeError("pedalboard khong kha dung - khong auto-sample VST3 duoc")
+        vst = PluginManager(extra_vst_dirs=extra_vst_dirs or []).load_vst(instrument_id)
+        if vst is None:
+            raise FileNotFoundError(f"Khong tim thay VSTi: {instrument_id} - hay Scan trong Plugin Manager truoc")
+        if preset_id or preset_path or preset_data_b64:
+            apply_preset_to_plugin(vst, preset_id=preset_id, preset_path=preset_path,
+                                   preset_data_b64=preset_data_b64)
+        def _render(note):
+            return _render_note(vst, note, sample_rate, duration, release, velocity)
+
    samples = []
    for note in range(low, high + 1, step):
-        frames = _render_note(vst, note, sample_rate, duration, release, velocity)
+        frames = _render(note)
        if frames is None:
            _log(f"note {note}: silent, bo qua")
            continue