From 73025fc38747acc5e541ea03692564058d4113cb Mon Sep 17 00:00:00 2001 From: 3dtours Date: Sun, 2 Aug 2026 21:44:49 +0700 Subject: [PATCH] =?UTF-8?q?fix:=20instrument=20SF3/Sonatina=20kh=C3=B4ng?= =?UTF-8?q?=20ph=C3=A1t=20=C3=A2m=20thanh=20-=20chuy=E1=BB=83n=20SF3->SF2,?= =?UTF-8?q?=20b=E1=BB=8F=20convert=20SF2->SF3,=20s=E1=BB=ADa=20converter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/api/v1/plugins.py | 29 +- app/core/soundfont_converter.py | 372 +++++++++++++++++++--- app/main.py | 13 +- app/static/js/services/soundfontPlayer.js | 29 +- wiki.md | 5 + 5 files changed, 387 insertions(+), 61 deletions(-) diff --git a/app/api/v1/plugins.py b/app/api/v1/plugins.py index e65f74a..4eb7fd1 100644 --- a/app/api/v1/plugins.py +++ b/app/api/v1/plugins.py @@ -142,12 +142,29 @@ async def download_soundfont_asset(sf_id: str): for base_dir in [UPLOAD_SF_DIR, SYSTEM_SF_DIR]: if not os.path.isdir(base_dir): continue - for ext in [".sf2", ".sf3"]: - for fname in os.listdir(base_dir): - fbase, fext = os.path.splitext(fname) - if fext.lower() == ext and fbase.lower() == clean_id.lower(): - full = os.path.join(base_dir, fname) - return FileResponse(full, media_type="application/octet-stream", filename=f"soundfont{ext}") + # Prefer SF2: the client FluidSynth WASM cannot decode SF3 (Ogg Vorbis) + # samples, so any SF3 would play silence in the browser. + for fname in os.listdir(base_dir): + fbase, fext = os.path.splitext(fname) + if fext.lower() == ".sf2" and fbase.lower() == clean_id.lower(): + full = os.path.join(base_dir, fname) + return FileResponse(full, media_type="application/octet-stream", filename="soundfont.sf2") + # Only an SF3 exists -> decompress it to a playable SF2 on demand (cached) + for fname in os.listdir(base_dir): + fbase, fext = os.path.splitext(fname) + if fext.lower() == ".sf3" and fbase.lower() == clean_id.lower(): + full = os.path.join(base_dir, fname) + try: + from app.core.soundfont_converter import SoundFontConverter + sf2_path = os.path.join(UPLOAD_SF_DIR, clean_id + ".sf2") + if os.path.exists(sf2_path) and os.path.getmtime(sf2_path) >= os.path.getmtime(full): + return FileResponse(sf2_path, media_type="application/octet-stream", filename="soundfont.sf2") + result = SoundFontConverter().sf3_to_sf2(full, sf2_path) + if result != full and os.path.exists(result): + return FileResponse(result, media_type="application/octet-stream", filename="soundfont.sf2") + except Exception as e: + print(f"[soundfont-download] SF3->SF2 conversion failed for {full}: {e}") + return FileResponse(full, media_type="application/octet-stream", filename="soundfont.sf3") raise HTTPException(status_code=404, detail="SoundFont asset not found") diff --git a/app/core/soundfont_converter.py b/app/core/soundfont_converter.py index 8452efe..31bf1bb 100644 --- a/app/core/soundfont_converter.py +++ b/app/core/soundfont_converter.py @@ -83,6 +83,35 @@ class SoundFontConverter: except Exception: return False + def _encode_sample_ogg(self, pcm: bytes, rate: int, tmp_dir: str) -> bytes: + """Encode a single mono 16-bit PCM slice to an Ogg Vorbis stream.""" + tmp_wav = os.path.join(tmp_dir, "sample_tmp.wav") + tmp_ogg = os.path.join(tmp_dir, "sample_tmp.ogg") + try: + with open(tmp_wav, "wb") as fw: + with wave.open(fw, "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(rate) + w.writeframes(pcm) + subprocess.run([ + "ffmpeg", "-y", "-i", tmp_wav, + "-c:a", "libvorbis", "-q:a", "3", + "-f", "ogg", tmp_ogg + ], capture_output=True, timeout=600, check=True) + with open(tmp_ogg, "rb") as fo: + return fo.read() + except Exception as e: + logger.error(f"Sample OGG encode failed: {e}") + return b"" + finally: + for p in [tmp_wav, tmp_ogg]: + try: + if os.path.exists(p): + os.remove(p) + except Exception: + pass + def _sf2_to_sf3_python(self, sf2_path: str, sf3_path: str) -> bool: has_ogg = self._check_ffmpeg_ogg() if not has_ogg: @@ -100,7 +129,7 @@ class SoundFontConverter: logger.warning("Not a valid SF2 file") return False - # Find smpl chunk recursively + # Find smpl chunk (PCM sample data) recursively smpl = _find_chunk(data, b"smpl") if smpl is None: logger.warning("No smpl chunk found in SF2") @@ -113,46 +142,103 @@ class SoundFontConverter: logger.warning("Sample data too small") return False - tmp_wav = sf3_path + ".tmp.wav" - tmp_ogg = sf3_path + ".tmp.ogg" + # Locate shdr (sample headers) inside the pdta LIST + pdta = _find_list_of_type(data, b"pdta", 12) + if not pdta: + logger.warning("No pdta LIST found") + return False + _, _, pdta_size, pdta_data_off = pdta + shdr = _find_chunk_in_list(data, b"shdr", pdta_data_off, pdta_size - 4) + if not shdr: + logger.warning("No shdr chunk found") + return False + _, _, shdr_size, shdr_data_off = shdr + if shdr_size <= 0 or shdr_size % 46 != 0: + logger.warning(f"Invalid shdr size {shdr_size}") + return False + n_samples = shdr_size // 46 + # Locate ifil (version) inside the INFO LIST + ifil_abs = None + info = _find_list_of_type(data, b"INFO", 12) + if info: + _, _, info_size, info_data_off = info + ifil = _find_chunk_in_list(data, b"ifil", info_data_off, info_size - 4) + if ifil: + ifil_abs = ifil[3] + + import tempfile + tmp_dir = tempfile.mkdtemp(prefix="sf3conv_") + ogg_parts = [] + new_shdr = bytearray() + byte_offset = 0 + ogg_bytes = 0 try: - # Write samples as WAV - with open(tmp_wav, "wb") as fw: - with wave.open(fw, "wb") as w: - w.setnchannels(1) - w.setsampwidth(2) - w.setframerate(44100) - w.writeframes(sample_data) + for i in range(n_samples): + base = shdr_data_off + i * 46 + # shdr layout: name[20] | start(u32) end(u32) loopstart(u32) loopend(u32) samplerate(i32) ... + start, end, startloop, endloop, rate = struct.unpack("= start and start * 2 < len(sample_data): + pcm = sample_data[start * 2:(end + 1) * 2] + if len(pcm) >= 4: + safe_rate = rate if 1000 < rate < 192000 else 44100 + ogg_stream = self._encode_sample_ogg(pcm, safe_rate, tmp_dir) + # SF3: start/end are byte offsets into the concatenated OGG stream. + # FluidSynth treats shdr `end` as EXCLUSIVE (reads [start..end-1]), + # so end = start + ogg length. + new_start = byte_offset + new_end = byte_offset + len(ogg_stream) + # OGG loop pointers are relative to the individual decompressed sample + new_sloop = (startloop - start) if (startloop > start and startloop <= end) else 0 + new_eloop = (endloop - start) if (endloop > start and endloop <= end) else 0 + # Mark the sample as Ogg Vorbis compressed (FLUID_SAMPLETYPE_OGG_VORBIS = 0x20) + new_stype = sampletype | 0x20 + new_shdr += data[base:base + 20] # sample name + new_shdr += struct.pack(" {len(ogg_data)} bytes ({compression:.0f}%)") - - ogg_padded = ogg_data if len(ogg_data) % 2 == 0 else ogg_data + b"\x00" - new_smpl_size = len(ogg_data) + ogg_padded = b"".join(ogg_parts) + if len(ogg_padded) % 2 == 1: + ogg_padded += b"\x00" + new_smpl_size = len(ogg_padded) old_padded = smpl_old_size + (1 if smpl_old_size % 2 == 1 else 0) - new_padded = len(ogg_padded) - delta = new_padded - old_padded + delta = len(ogg_padded) - old_padded # Rebuild file: replace smpl chunk and update all sizes out = bytearray() - out.extend(data[:smpl_head_off]) # up to smpl chunk header + out.extend(data[:smpl_head_off]) # up to smpl chunk header (excl. id) + out.extend(b"smpl") # chunk id (required for valid SF3) out.extend(struct.pack(" len(rest): + logger.warning("shdr not found after smpl chunk") + return False + rest[shdr_in_rest:shdr_in_rest + shdr_size] = new_shdr + out.extend(rest) data_out = bytes(out) + # SF3 requires version 3.0 so FluidSynth treats it as an SF3 file + if ifil_abs is not None and ifil_abs + 4 <= len(data_out): + data_out = data_out[:ifil_abs] + struct.pack(" {ogg_bytes} bytes OGG ({100 * (1 - ogg_bytes / max(len(sample_data), 1)):.0f}% smaller)") return True except subprocess.TimeoutExpired: @@ -185,12 +271,44 @@ class SoundFontConverter: traceback.print_exc() return False finally: - for p in [tmp_wav, tmp_ogg]: + try: + import shutil + shutil.rmtree(tmp_dir, ignore_errors=True) + except Exception: + pass + + @staticmethod + def _sf3_plays_audio(path: str) -> bool: + """Verify a SoundFont actually loads and renders audible audio (guards + against shipping malformed SF3 files that silently play nothing).""" + if not os.path.exists(path): + return False + try: + import fluidsynth + import numpy as np + fl = fluidsynth.Synth() + try: + h = fl.sfload(path) + if h < 0: + return False + fl.program_select(0, h, 0, 0) + fl.noteon(0, 60, 100) + frames = 8820 # 0.2s + buf = np.zeros(frames * 2, dtype=np.float32) + fluidsynth._fl.fluid_synth_write_float( + fl.synth, frames, buf.ctypes.data, 0, 1, + buf.ctypes.data + frames * 4, 0, 1 + ) + fl.noteoff(0, 60) + rms = float(np.sqrt(np.mean(buf ** 2))) + return rms > 1e-4 + finally: try: - if os.path.exists(p): - os.remove(p) + fl.delete() except Exception: pass + except Exception: + return False @staticmethod def _find_sf3_converter(): @@ -208,9 +326,18 @@ class SoundFontConverter: sf3_path = os.path.splitext(sf2_path)[0] + ".sf3" + # Reuse a working SF3 if it is newer than the SF2 AND actually plays + # audio. Malformed SF3s (e.g. produced by an older converter) are + # re-converted automatically instead of being shipped silently broken. if os.path.exists(sf3_path) and os.path.getmtime(sf3_path) >= os.path.getmtime(sf2_path): - logger.info(f"SF3 already up-to-date: {sf3_path}") - return sf3_path + if self._sf3_plays_audio(sf3_path): + logger.info(f"SF3 already up-to-date: {sf3_path}") + return sf3_path + logger.warning(f"Existing SF3 does not play audio, re-converting: {sf3_path}") + try: + os.remove(sf3_path) + except Exception: + pass converter = self._find_sf3_converter() try: @@ -228,9 +355,16 @@ class SoundFontConverter: if converter == "python": if self._sf2_to_sf3_python(sf2_path, sf3_path) and os.path.exists(sf3_path): - size_mb = os.path.getsize(sf3_path) / (1024 * 1024) - logger.info(f"Created SF3: {sf3_path} ({size_mb:.2f} MB)") - return sf3_path + if self._sf3_plays_audio(sf3_path): + size_mb = os.path.getsize(sf3_path) / (1024 * 1024) + logger.info(f"Created SF3: {sf3_path} ({size_mb:.2f} MB)") + return sf3_path + # Conversion produced a broken file — never ship it + logger.warning(f"Converted SF3 failed audio verification, removing: {sf3_path}") + try: + os.remove(sf3_path) + except Exception: + pass logger.warning(f"Python converter failed for {sf2_path}, returning SF2 path") return sf2_path @@ -242,6 +376,165 @@ class SoundFontConverter: logger.error(f"Error converting {sf2_path}: {e}") return sf2_path + def sf3_to_sf2(self, sf3_path: str, sf2_path: str = None) -> str: + """Convert an SF3 (Ogg Vorbis samples) soundfont into a playable SF2. + + The client FluidSynth WASM cannot decode Ogg Vorbis/SF3 samples, so any + SF3 soundfont (converted or uploaded) plays silence. Decompressing to + SF2 makes every instrument audible again. + """ + if not os.path.exists(sf3_path): + raise FileNotFoundError(f"Source SF3 file not found: {sf3_path}") + + try: + with open(sf3_path, "rb") as f: + data = f.read() + except Exception as e: + raise IOError(f"Cannot read {sf3_path}: {e}") + + if len(data) < 12 or data[:4] != b"RIFF" or data[8:12] != b"sfbk": + raise ValueError(f"Not a valid SoundFont file: {sf3_path}") + + smpl = _find_chunk(data, b"smpl") + if not smpl: + raise ValueError("No smpl chunk found") + smpl_head_off, _, smpl_old_size, smpl_data_off = smpl + sample_data = data[smpl_data_off:smpl_data_off + smpl_old_size] + + pdta = _find_list_of_type(data, b"pdta", 12) + if not pdta: + raise ValueError("No pdta LIST found") + _, _, pdta_size, pdta_data_off = pdta + shdr = _find_chunk_in_list(data, b"shdr", pdta_data_off, pdta_size - 4) + if not shdr: + raise ValueError("No shdr chunk found") + _, _, shdr_size, shdr_data_off = shdr + if shdr_size <= 0 or shdr_size % 46 != 0: + raise ValueError(f"Invalid shdr size {shdr_size}") + n_samples = shdr_size // 46 + + ifil_abs = None + info = _find_list_of_type(data, b"INFO", 12) + if info: + _, _, info_size, info_data_off = info + ifil = _find_chunk_in_list(data, b"ifil", info_data_off, info_size - 4) + if ifil: + ifil_abs = ifil[3] + + import tempfile + tmp_dir = tempfile.mkdtemp(prefix="sf2conv_") + pcm_parts = [] + new_shdr = bytearray() + frame_offset = 0 + total_pcm_bytes = 0 + try: + for i in range(n_samples): + base = shdr_data_off + i * 46 + name = data[base:base + 20] + start, end, loopstart, loopend, rate = struct.unpack(" start and start < len(sample_data): + # FluidSynth reads the OGG region as [start..end-1] + ogg = sample_data[start:min(end, len(sample_data))] + if len(ogg) >= 4 and ogg[:4] == b"OggS": + pcm = self._decode_sample_ogg(ogg, tmp_dir) + frames = len(pcm) // 2 + new_start = frame_offset + new_end = frame_offset + frames + # SF3 loop points are relative to the decompressed sample; SF2 needs absolute + new_loopstart = loopstart + new_start if (loopstart or loopend) else 0 + new_loopend = loopend + new_start if (loopstart or loopend) else 0 + # Clear the Ogg Vorbis flag; keep mono/left/right/linked flags + new_stype = sampletype & ~0x20 + new_shdr += name + new_shdr += struct.pack(" len(rest): + logger.warning("shdr not found after smpl chunk") + return sf3_path + rest[shdr_in_rest:shdr_in_rest + shdr_size] = new_shdr + out.extend(rest) + + data_out = bytes(out) + + # Back to SF2 version 2.01 + if ifil_abs is not None and ifil_abs + 4 <= len(data_out): + data_out = data_out[:ifil_abs] + struct.pack(" SF2: {sf2_path} ({os.path.getsize(sf2_path) / 1048576:.1f} MB)") + return sf2_path + finally: + try: + import shutil + shutil.rmtree(tmp_dir, ignore_errors=True) + except Exception: + pass + + def _decode_sample_ogg(self, ogg: bytes, tmp_dir: str) -> bytes: + """Decode an Ogg Vorbis stream to mono 16-bit PCM; returns PCM bytes.""" + tmp_ogg = os.path.join(tmp_dir, "sample.ogg") + tmp_pcm = os.path.join(tmp_dir, "sample.pcm") + try: + with open(tmp_ogg, "wb") as fo: + fo.write(ogg) + r = subprocess.run( + ["ffmpeg", "-y", "-v", "error", "-i", tmp_ogg, "-f", "s16le", "-ac", "1", tmp_pcm], + capture_output=True, timeout=600) + if r.returncode != 0: + logger.warning(f"OGG decode failed: {r.stderr.decode(errors='replace')[:120]}") + return b"" + with open(tmp_pcm, "rb") as fp: + return fp.read() + except Exception as e: + logger.warning(f"OGG decode error: {e}") + return b"" + finally: + for p in [tmp_ogg, tmp_pcm]: + try: + if os.path.exists(p): + os.remove(p) + except Exception: + pass + def batch_convert_all(self): for sdir in self.target_dirs: if not os.path.isdir(sdir): @@ -249,3 +542,4 @@ class SoundFontConverter: for fname in sorted(os.listdir(sdir)): if fname.lower().endswith(".sf2"): self.convert_sf2_to_sf3(os.path.join(sdir, fname)) + diff --git a/app/main.py b/app/main.py index 42249ed..c3280ac 100644 --- a/app/main.py +++ b/app/main.py @@ -1,4 +1,4 @@ -import os, threading +import os from fastapi import FastAPI from fastapi.responses import HTMLResponse from fastapi.staticfiles import StaticFiles @@ -16,7 +16,6 @@ from app.api.v1.ai_presets import router as ai_presets_router from app.api.v1.plugins import router as plugins_router from app.api.v1.media import router as media_router from app.core.auth import seed_admin -from app.core.soundfont_converter import SoundFontConverter from app.core.soundfont_scanner import SoundFontAutoScanner # Ensure storage directories exist @@ -63,12 +62,10 @@ async def startup_seed_admin(): @app.on_event("startup") async def startup_convert_soundfonts(): - def _run(): - try: - SoundFontConverter().batch_convert_all() - except Exception as e: - print(f"[Startup] SoundFont conversion error: {e}") - threading.Thread(target=_run, daemon=True).start() + # SF2 -> SF3 conversion is disabled: the client FluidSynth WASM cannot decode + # Ogg Vorbis (SF3) samples, so converted SF3s would play silence. The download + # endpoint serves SF2 when available and converts SF3 -> SF2 on demand instead. + pass @app.on_event("startup") async def startup_sf_scanner(): diff --git a/app/static/js/services/soundfontPlayer.js b/app/static/js/services/soundfontPlayer.js index 8e4b6d7..8cf0c90 100644 --- a/app/static/js/services/soundfontPlayer.js +++ b/app/static/js/services/soundfontPlayer.js @@ -206,16 +206,29 @@ try { var cache = window.SonicSFStorage; var buf = cache ? await cache.getBuffer(sfId) : null; - if (!buf) { - var url = "/api/v1/plugins/soundfonts/download/" + encodeURIComponent(sfId) + "?t=" + Date.now(); - var resp = await fetch(url); - if (!resp.ok) { - console.warn("[SonicSF] SoundFont not found:", sfId); - return false; + if (buf) { + var cachedOk = this._tryLoadSFL(buf, '.sf3'); + if (cachedOk === -1) cachedOk = this._tryLoadSFL(buf, '.sf2'); + if (cachedOk !== -1) { + _sfHandleMap.set(sfId, cachedOk); + _currentSfId = sfId; + _loadedFonts[sfId] = true; + console.log("[SonicSF] SoundFont loaded from cache:", sfId, "handle:", cachedOk); + return true; } - buf = await resp.arrayBuffer(); - if (cache) await cache.saveBuffer(sfId, buf); + // Stale/corrupt cache (e.g. old SF3 buffers the WASM can't + // decode) — drop it and re-download from the server. + console.warn("[SonicSF] Cached SoundFont unplayable, re-downloading:", sfId); + try { await cache.saveBuffer(sfId, null); } catch (e2) {} } + var url = "/api/v1/plugins/soundfonts/download/" + encodeURIComponent(sfId) + "?t=" + Date.now(); + var resp = await fetch(url); + if (!resp.ok) { + console.warn("[SonicSF] SoundFont not found:", sfId); + return false; + } + buf = await resp.arrayBuffer(); + if (cache) await cache.saveBuffer(sfId, buf); var sfHandle = this._tryLoadSFL(buf, '.sf3'); if (sfHandle === -1) { console.warn("[SonicSF] sfload .sf3 failed, trying .sf2 for", sfId); diff --git a/wiki.md b/wiki.md index faaf502..4b91376 100644 --- a/wiki.md +++ b/wiki.md @@ -1184,3 +1184,8 @@ - **Tóm tắt thay đổi:** Media Explorer preview canvas dùng React `onWheel={handleCanvasWheel}` có `e.preventDefault()` — React gắn `wheel` passive tại root nên Chrome log lỗi "Unable to preventDefault inside passive event listener invocation" và page vẫn scroll. Thay bằng native listener: `useEffect` + `canvasRef` + `addEventListener('wheel', h, { passive: false })` qua `handleCanvasWheelRef` (luôn gọi handler mới nhất), bỏ prop `onWheel` khỏi ``. Giờ block scroll thật + hết lỗi console. - **Các file ảnh hưởng:** `app/static/js/app.jsx`, `app/static/js/app.precompiled.js` - **Ghi chú/Test (nếu có):** `npm run build` OK; bundle: `handleCanvasWheelRef` có mặt, `onWheel:handleCanvasWheel` đã bỏ. Scroll chuột trên canvas preview (có file selected) → zoom hoạt động, không lỗi passive. Lưu ý: các onWheel tương tự ở MixerStrip/MasterStrip/TrackStripConsole (pan dial, fader) vẫn còn cảnh báo này — chưa sửa. + +### [2026-08-02 21:40] Task: Instrument Sonatina soundfont không phát âm thanh +- **Tóm tắt thay đổi:** Nguyên nhân gốc: FluidSynth WASM (soundfontPlayer) KHÔNG decode được mẫu Ogg Vorbis/SF3 — mọi sample có bit OGG_VORBIS bị từ chối ("unknown flags... unsupported compression") → chọn instrument từ soundfont .sf3 (Sonatina đang ở dạng SF3 do startup convert) → câm hoàn toàn. Ngoài ra converter SF2→SF3 cũ còn 2 bug: (1) thiếu chunk id 'smpl' khi dựng lại file → sfload -1; (2) offset đọc shdr sai (đọc [end,loopstart,loopend,rate] thay vì [start,end,...]) → smpl rỗng. Fix: (1) `sf3_to_sf2()` + `_decode_sample_ogg()` chuyển SF3→SF2 (giải nén OGG về PCM, đúng semantics end-exclusive, loop absolute, xóa bit OGG) — đã verify phát âm thanh cả native lẫn WASM; (2) endpoint download ưu tiên .sf2, nếu chỉ có .sf3 thì chuyển SF3→SF2 on-demand (cache vào upload dir); (3) vô hiệu startup convert SF2→SF3 (`main.py`) — client không chơi được SF3; (4) client `loadSoundFont` nếu cache IndexedDB cũ chứa buffer hỏng thì xóa cache + tải lại; (5) sửa `_sf2_to_sf3_python` (smpl id, shdr offset đúng, end-exclusive, version 3.0, bit OGG, loop relative) + gate `_sf3_plays_audio` xóa SF3 không phát được. +- **Các file ảnh hưởng:** `app/core/soundfont_converter.py`, `app/api/v1/plugins.py`, `app/main.py`, `app/static/js/services/soundfontPlayer.js` +- **Ghi chú/Test (nếu có):** Test Node/WASM + native: SF2 gốc rms 0.007 (OK); SF3 convert mới sfload=1 nhưng câm (WASM từ chối OGG); SF3→SF2 convert ra file 52.37MB phát rms 0.007-0.02 (native + WASM). Test endpoint mô phỏng: upload SF3-only → download trả SF2 phát được. `pytest tests/test_vst_engine.py` 12 passed. Lưu ý: scipy thiếu nên test_plugin_api không import được (môi trường dev).