From 0acc3ee7ee04051427754fe2f6600038f60dfe59 Mon Sep 17 00:00:00 2001 From: locphamtran Date: Sun, 23 Aug 2026 11:44:54 +0700 Subject: [PATCH] =?UTF-8?q?fix(fx-rt):=20burst=204-block=20input=20batch?= =?UTF-8?q?=20qua=20bridge=20=E2=80=94=20h=E1=BA=BFt=20lag/gi=E1=BA=ADt/cr?= =?UTF-8?q?ackle=20master=20FX?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - worklet gom 1024 mẫu (4 block) vào 1 frame interleaved, main thread tách 4 WS frame 512 floats - bridge take=min(avail,4) xử lí 1 call process cho burst → overhead VST3 amortize 4 lần - engine _pump_output pace theo deadline thay vì sleep cố định (outRead 131→187.5/s) - FX_RT_ROUNDTRIP_SAMPLES 512→1792 (RTT batching ~7 block) --- app/api/v1/plugins.py | 13 +++++- app/static/js/app.jsx | 59 ++++++++++++++++----------- app/static/js/app.precompiled.js | 19 ++++++--- app/static/js/sf-fx-realtime.js | 24 +++++++---- app/templates/index.html | 2 +- native_bridge/src/RealtimeFxLoop.cpp | 43 ++++++++++++------- src-tauri/binaries/fx_vst_bridge.exe | Bin 312320 -> 345600 bytes 7 files changed, 108 insertions(+), 52 deletions(-) diff --git a/app/api/v1/plugins.py b/app/api/v1/plugins.py index 638f8cd..1fe1f29 100644 --- a/app/api/v1/plugins.py +++ b/app/api/v1/plugins.py @@ -1071,6 +1071,7 @@ async def ws_fx_realtime(websocket: WebSocket, session_id: str): # de bu delay dry-bypass path (app.jsx fxRtApplyPdc). _last_lat = None try: + _loop = asyncio.get_event_loop() while True: try: _lats = sess.get_latencies() @@ -1088,9 +1089,17 @@ async def ws_fx_realtime(websocket: WebSocket, session_id: str): if not blks: await asyncio.sleep(0.002) continue - for blk in blks: + # Pace theo deadline (không sleep cố định sau mỗi send): thời + # gian send_bytes (WS → browser) CỘNG sleep cố định làm chu kì + # mỗi block > block_sec → engine drain chậm hơn realtime + # (đo: outRead 131/s thay vì 187.5/s → worklet zero-fill crackle). + # Gửi mỗi block cách nhau đúng block_sec kể từ mốc t0. + _t0 = _loop.time() + for _i, blk in enumerate(blks): await websocket.send_bytes(blk) - await asyncio.sleep(block_sec) + _dt = _t0 + (_i + 1) * block_sec - _loop.time() + if _dt > 0: + await asyncio.sleep(_dt) except Exception: pass pump_task = asyncio.create_task(_pump_output()) diff --git a/app/static/js/app.jsx b/app/static/js/app.jsx index a1ca006..bce0dd4 100644 --- a/app/static/js/app.jsx +++ b/app/static/js/app.jsx @@ -1711,7 +1711,7 @@ async function fxRtStart() { fxRt.sessionId = st.session_id; fxRtGuiPushShm(); try { - await ctx.audioWorklet.addModule(`${API_BASE_URL}/static/js/sf-fx-realtime.js?v=202608212200`); + await ctx.audioWorklet.addModule(`${API_BASE_URL}/static/js/sf-fx-realtime.js?v=202608212300`); fxRt.node = new AudioWorkletNode(ctx, 'sf-fx-realtime', { numberOfInputs: 1, numberOfOutputs: 1, outputChannelCount: [2] }); const u = new URL(API_BASE_URL); const wsProto = u.protocol === 'https:' ? 'wss://' : 'ws://'; @@ -1720,7 +1720,11 @@ async function fxRtStart() { const ws = new WebSocket(wsUrl); fxRt.ws = ws; ws.binaryType = 'arraybuffer'; - ws.onopen = () => { fxRt.active = true; try { fxRtPushAllParams(); } catch (e) {} }; + ws.onopen = () => { + fxRt.active = true; + fxRt.inQueue = []; // bỏ audio stale sản xuất trước khi WS open (~4s load VST) — hết lag khởi đầu + try { fxRtPushAllParams(); } catch (e) {} + }; ws.onmessage = (ev) => { // Gap 8 (PDC realtime): text frame {cmd:'latency', total} tu server -> // cap nhat delay dry-bypass path (bu latency chain VST3 realtime). @@ -1752,20 +1756,25 @@ async function fxRtStart() { return; } if (d.type !== 'in') return; - if (fxRt.inQueue.length > 64) fxRt.inQueue.shift(); - fxRt.inQueue.push(d.data); + // Worklet gom 4 block (1024 mẫu) vào 1 frame → tách 4 WS frame 512 floats + // ở đây. Bridge VST3 nhận input dồn → take=4 → batch 1 call process 1024 + // mẫu (~3.5-4.75ms/block) → đủ 187.5/s (take=1: 5.5ms > budget 5.33ms → + // drop → crackle). KHÔNG dùng f.subarray().buffer — nó gửi cả backing + // buffer 8192B; slice() copy đúng 2048B. + if (ws.readyState === 1) { + try { + const f = new Float32Array(d.data); + if (f.length > 512) { + for (let i = 0; i < f.length; i += 512) ws.send(f.slice(i, i + 512).buffer); + } else { + ws.send(d.data); + } + } catch (e) {} + } }; // Pace input: tối đa 1 block/kỳ (~5.8ms). Gửi dồn theo burst (main thread bị // rAF renderFrame block khi mở MASTERING PANEL) → SHM ring 4-slot tràn → mất // block → ngắt quãng. - const blockMs = Math.max(4, Math.round((256 / (ctx.sampleRate || 44100)) * 1000)); - if (fxRt.pumpTimer) clearInterval(fxRt.pumpTimer); - fxRt.pumpTimer = setInterval(() => { - const ws = fxRt.ws; - if (!ws || ws.readyState !== 1 || !fxRt.inQueue.length) return; - const d = fxRt.inQueue.shift(); - try { ws.send(d); } catch (e) { fxRt.inQueue.unshift(d); } - }, blockMs); // Phase 4.2: fader SAU worklet — worklet chèn giữa outputAnalyser (chain // output) và output (fader/destination). output → analyser/splitter giữ // nguyên (meter đo âm SAU fader). @@ -1812,7 +1821,8 @@ function fxRtStop(silent) { // qua worklet → lệch pha với mastered. fxRtApplyPdc trễ dry path đúng lượng // để align. FX_RT_ROUNDTRIP_SAMPLES là ước lượng (~2 block 256 @44.1k) — // ponytail: đo RTT thật (tag seq trên block) khi cần sample-accurate. -const FX_RT_ROUNDTRIP_SAMPLES = 512; +// Batching 4 block làm RTT ~7 block (~40ms @48k) → 1792. +const FX_RT_ROUNDTRIP_SAMPLES = 1792; function fxRtApplyPdc() { const ctx = getAudioContext(); const bus = masterBus; @@ -23652,7 +23662,7 @@ const App = () => { fxRtTracks[key] = entry; fxRtGuiPushShm(); try { - await ctx.audioWorklet.addModule(`${API_BASE_URL}/static/js/sf-fx-realtime.js?v=202608212200`); + await ctx.audioWorklet.addModule(`${API_BASE_URL}/static/js/sf-fx-realtime.js?v=202608212300`); const node = new AudioWorkletNode(ctx, 'sf-fx-realtime', { numberOfInputs: 1, numberOfOutputs: 1, outputChannelCount: [2] }); entry.node = node; const u = new URL(API_BASE_URL); @@ -23663,6 +23673,7 @@ const App = () => { ws.binaryType = 'arraybuffer'; ws.onopen = () => { entry.ready = true; + entry.inQueue = []; // bỏ audio stale trước khi WS open — hết lag khởi đầu fxRtTrackSplice(trackId, path); try { (entry.paramsBySlot || []).forEach((pl, slot) => pl.forEach(p => ws.send(JSON.stringify({ cmd: 'set_param', slot, key: p.key, value: p.value })))); @@ -23687,16 +23698,18 @@ const App = () => { return; } if (d.type !== 'in') return; - if (entry.inQueue.length > 64) entry.inQueue.shift(); - entry.inQueue.push(d.data); + // Burst 4 block (1024 mẫu) → tách 4 WS frame 512 floats → bridge batch 4. + if (ws.readyState === 1) { + try { + const f = new Float32Array(d.data); + if (f.length > 512) { + for (let i = 0; i < f.length; i += 512) ws.send(f.slice(i, i + 512).buffer); + } else { + ws.send(d.data); + } + } catch (e) {} + } }; - const blockMs = Math.max(4, Math.round((256 / (ctx.sampleRate || 44100)) * 1000)); - entry.pumpTimer = setInterval(() => { - const ws = entry.ws; - if (!ws || ws.readyState !== 1 || !entry.inQueue.length) return; - const d = entry.inQueue.shift(); - try { ws.send(d); } catch (e) { entry.inQueue.unshift(d); } - }, blockMs); } catch (e) { console.warn('[FX RT track] setup fail', trackId, path, e); fxRtTrackStop(trackId, path, true); diff --git a/app/static/js/app.precompiled.js b/app/static/js/app.precompiled.js index 38757fd..6bc2747 100644 --- a/app/static/js/app.precompiled.js +++ b/app/static/js/app.precompiled.js @@ -281,12 +281,18 @@ return(ms.chain||[]).filter(s=>s&&s.active!==false&&(s.type!=='vst3'||!ms.vstByp const FX_RT_PARAM_MAP={eq:[['g1','eqLowGain'],['g2','eqMid1Gain'],['g3','eqMid2Gain'],['g4','eqHighGain']],compressor:[['threshold','compThreshold'],['ratio','compRatio'],['makeup','compMakeup']],limiter:[['ceiling','limThreshold'],['mode','limMode'],['lookahead_ms','limLookahead']],exciter:[['drive','excDrive']],rebalance:[['mid','rebalMid'],['side','rebalSide']],imager:[['w1','w1'],['w2','w2'],['w3','w3'],['w4','w4']],maximizer:[['boost_db','maxGain'],['soft_clip','maxSoftClip'],['upward','maxUpward'],['ceiling_db','ceiling']]};function fxRtEntryParams(entry,ms){const out=[];if(!entry||entry.type==='vst3')return out;const src=ms||window.currentMasteringSettings||{};const push=(k,v)=>{const n=Number(v);if(isFinite(n))out.push({key:k,value:n});};const map=FX_RT_PARAM_MAP[entry.type]||[];map.forEach(([cppKey,jsKey])=>{if(entry.params&&entry.params[cppKey]!==undefined)push(cppKey,entry.params[cppKey]);else if(entry.params&&entry.params[jsKey]!==undefined)push(jsKey,entry.params[jsKey]);else if(src[jsKey]!==undefined)push(cppKey,src[jsKey]);});if(entry.type==='eqpro'){if(entry.params&&entry.params.amount!==undefined)push('amount',entry.params.amount);const bands=entry.params&&entry.params.bands||[];bands.forEach((b,i)=>{['freq','gain','q'].forEach(k=>{if(b&&b[k]!==undefined)push('band'+i+'_'+k,b[k]);});});}return out;}function fxRtPushAllParams(){if(!fxRt||!fxRt.ws||fxRt.ws.readyState!==1)return;const chain=fxRtChain();chain.forEach((s,i)=>{fxRtEntryParams(s,null).forEach(p=>{try{fxRt.ws.send(JSON.stringify({cmd:'set_param',slot:i,key:p.key,value:p.value}));}catch(e){}});});}async function fxRtStart(){const chain=fxRtChain();const ck=fxRtChainKey(chain);if(fxRt.active){// masterBus bị rebuild (recovery/export) → node thuộc graph cũ → restart. if(!fxRt.node||fxRt.node.context!==getAudioContext())fxRtStop(true);// Chain VST thay đổi (insert/remove/bypass/reorder trong MASTER FX CHAIN) // → stop + start lại để bridge load đúng chuỗi FX mới. -else if(fxRt.chainKey!==ck)fxRtStop(true);else return;}if(!chain.length)return;const ctx=getAudioContext();if(!ctx||!ctx.audioWorklet||!masterBus||!masterBus.output||!masterBus.analyser)return;const chainPayload=chain.map(s=>s.type==='vst3'?{type:'vst3',path:s.path,name:s.name||s.path,preset_b64:s.preset_b64||'',bypass:false}:{type:'builtin',id:s.type,params:s.params&&Object.keys(s.params).length?s.params:{},bypass:false});let st;try{st=await window.SonicAPI.fxRealtimeStart({fx_chain:chainPayload,sample_rate:ctx.sampleRate||44100});}catch(e){console.warn('[FX RT] start fail:',e);showToast('Không khởi động VST FX realtime: '+(e.message||e),'warning');return;}fxRt.sessionId=st.session_id;fxRtGuiPushShm();try{await ctx.audioWorklet.addModule(`${API_BASE_URL}/static/js/sf-fx-realtime.js?v=202608212200`);fxRt.node=new AudioWorkletNode(ctx,'sf-fx-realtime',{numberOfInputs:1,numberOfOutputs:1,outputChannelCount:[2]});const u=new URL(API_BASE_URL);const wsProto=u.protocol==='https:'?'wss://':'ws://';const tok=localStorage.getItem('sonic_token')||'';const wsUrl=`${wsProto}${u.host}${st.ws_url}${tok?'?token='+encodeURIComponent(tok):''}`;const ws=new WebSocket(wsUrl);fxRt.ws=ws;ws.binaryType='arraybuffer';ws.onopen=()=>{fxRt.active=true;try{fxRtPushAllParams();}catch(e){}};ws.onmessage=ev=>{// Gap 8 (PDC realtime): text frame {cmd:'latency', total} tu server -> +else if(fxRt.chainKey!==ck)fxRtStop(true);else return;}if(!chain.length)return;const ctx=getAudioContext();if(!ctx||!ctx.audioWorklet||!masterBus||!masterBus.output||!masterBus.analyser)return;const chainPayload=chain.map(s=>s.type==='vst3'?{type:'vst3',path:s.path,name:s.name||s.path,preset_b64:s.preset_b64||'',bypass:false}:{type:'builtin',id:s.type,params:s.params&&Object.keys(s.params).length?s.params:{},bypass:false});let st;try{st=await window.SonicAPI.fxRealtimeStart({fx_chain:chainPayload,sample_rate:ctx.sampleRate||44100});}catch(e){console.warn('[FX RT] start fail:',e);showToast('Không khởi động VST FX realtime: '+(e.message||e),'warning');return;}fxRt.sessionId=st.session_id;fxRtGuiPushShm();try{await ctx.audioWorklet.addModule(`${API_BASE_URL}/static/js/sf-fx-realtime.js?v=202608212300`);fxRt.node=new AudioWorkletNode(ctx,'sf-fx-realtime',{numberOfInputs:1,numberOfOutputs:1,outputChannelCount:[2]});const u=new URL(API_BASE_URL);const wsProto=u.protocol==='https:'?'wss://':'ws://';const tok=localStorage.getItem('sonic_token')||'';const wsUrl=`${wsProto}${u.host}${st.ws_url}${tok?'?token='+encodeURIComponent(tok):''}`;const ws=new WebSocket(wsUrl);fxRt.ws=ws;ws.binaryType='arraybuffer';ws.onopen=()=>{fxRt.active=true;fxRt.inQueue=[];// bỏ audio stale sản xuất trước khi WS open (~4s load VST) — hết lag khởi đầu +try{fxRtPushAllParams();}catch(e){}};ws.onmessage=ev=>{// Gap 8 (PDC realtime): text frame {cmd:'latency', total} tu server -> // cap nhat delay dry-bypass path (bu latency chain VST3 realtime). -if(typeof ev.data==='string'){try{const d=JSON.parse(ev.data);if(d&&d.cmd==='latency'&&typeof d.total==='number'){fxRt.latencyTotal=d.total;fxRtApplyPdc();}}catch(e){}return;}if(!(ev.data instanceof ArrayBuffer)||!fxRt.node)return;const f=new Float32Array(ev.data);const half=f.length/2;const l=new Float32Array(half);const r=new Float32Array(half);for(let i=0;ifxRtStop(true);ws.onerror=()=>{try{ws.close();}catch(e){}};fxRt.node.port.onmessage=ev=>{const d=ev.data;if(!d||!fxRt.ws)return;if(d.type==='set_param'){try{fxRt.ws.send(JSON.stringify({cmd:'set_param',slot:d.slot,key:d.key,value:d.value}));}catch(e){}return;}if(d.type!=='in')return;if(fxRt.inQueue.length>64)fxRt.inQueue.shift();fxRt.inQueue.push(d.data);};// Pace input: tối đa 1 block/kỳ (~5.8ms). Gửi dồn theo burst (main thread bị +if(typeof ev.data==='string'){try{const d=JSON.parse(ev.data);if(d&&d.cmd==='latency'&&typeof d.total==='number'){fxRt.latencyTotal=d.total;fxRtApplyPdc();}}catch(e){}return;}if(!(ev.data instanceof ArrayBuffer)||!fxRt.node)return;const f=new Float32Array(ev.data);const half=f.length/2;const l=new Float32Array(half);const r=new Float32Array(half);for(let i=0;ifxRtStop(true);ws.onerror=()=>{try{ws.close();}catch(e){}};fxRt.node.port.onmessage=ev=>{const d=ev.data;if(!d||!fxRt.ws)return;if(d.type==='set_param'){try{fxRt.ws.send(JSON.stringify({cmd:'set_param',slot:d.slot,key:d.key,value:d.value}));}catch(e){}return;}if(d.type!=='in')return;// Worklet gom 4 block (1024 mẫu) vào 1 frame → tách 4 WS frame 512 floats +// ở đây. Bridge VST3 nhận input dồn → take=4 → batch 1 call process 1024 +// mẫu (~3.5-4.75ms/block) → đủ 187.5/s (take=1: 5.5ms > budget 5.33ms → +// drop → crackle). KHÔNG dùng f.subarray().buffer — nó gửi cả backing +// buffer 8192B; slice() copy đúng 2048B. +if(ws.readyState===1){try{const f=new Float32Array(d.data);if(f.length>512){for(let i=0;i{const ws=fxRt.ws;if(!ws||ws.readyState!==1||!fxRt.inQueue.length)return;const d=fxRt.inQueue.shift();try{ws.send(d);}catch(e){fxRt.inQueue.unshift(d);}},blockMs);// Phase 4.2: fader SAU worklet — worklet chèn giữa outputAnalyser (chain +// Phase 4.2: fader SAU worklet — worklet chèn giữa outputAnalyser (chain // output) và output (fader/destination). output → analyser/splitter giữ // nguyên (meter đo âm SAU fader). try{masterBus.outputAnalyser.disconnect(masterBus.output);}catch(e){}masterBus.outputAnalyser.connect(fxRt.node);fxRt.node.connect(masterBus.output);fxRt.chainKey=ck;}catch(e){console.warn('[FX RT] setup fail:',e);fxRtStop(true);}}function fxRtStop(silent){if(fxRt.pumpTimer){clearInterval(fxRt.pumpTimer);fxRt.pumpTimer=null;}fxRt.inQueue=[];if(!fxRt.active&&!fxRt.ws&&!fxRt.node&&!fxRt.sessionId)return;try{if(fxRt.node){fxRt.node.disconnect();if(masterBus&&masterBus.outputAnalyser&&masterBus.output){try{masterBus.outputAnalyser.disconnect(masterBus.output);}catch(e){}try{masterBus.outputAnalyser.connect(masterBus.output);}catch(e){}}fxRt.node.port.onmessage=null;fxRt.node=null;}}catch(e){}try{if(fxRt.ws){fxRt.ws.onclose=null;fxRt.ws.onmessage=null;try{fxRt.ws.close();}catch(e){}fxRt.ws=null;}}catch(e){}const sid=fxRt.sessionId;fxRt.sessionId=null;fxRtGuiPushShm();fxRt.active=false;fxRt.chainKey=null;fxRt.latencyTotal=0;fxRtApplyPdc();if(sid){try{window.SonicAPI.fxRealtimeStop({session_id:sid}).catch(()=>{});}catch(e){}}}// ── Gap 8: PDC realtime — bù delay dry-bypass path (master chain) ────────── @@ -295,7 +301,8 @@ try{masterBus.outputAnalyser.disconnect(masterBus.output);}catch(e){}masterBus.o // qua worklet → lệch pha với mastered. fxRtApplyPdc trễ dry path đúng lượng // để align. FX_RT_ROUNDTRIP_SAMPLES là ước lượng (~2 block 256 @44.1k) — // ponytail: đo RTT thật (tag seq trên block) khi cần sample-accurate. -const FX_RT_ROUNDTRIP_SAMPLES=512;function fxRtApplyPdc(){const ctx=getAudioContext();const bus=masterBus;if(!ctx||!bus||!bus.pdcDelay)return;const l=fxRt&&fxRt.active?FX_RT_ROUNDTRIP_SAMPLES+(fxRt.latencyTotal||0):0;try{bus.pdcDelay.delayTime.setTargetAtTime(l/(ctx.sampleRate||44100),ctx.currentTime,0.05);}catch(e){}}// ── Realtime VST FX per-track (SHM bridge) — mọi luồng track qua bridge ───── +// Batching 4 block làm RTT ~7 block (~40ms @48k) → 1792. +const FX_RT_ROUNDTRIP_SAMPLES=1792;function fxRtApplyPdc(){const ctx=getAudioContext();const bus=masterBus;if(!ctx||!bus||!bus.pdcDelay)return;const l=fxRt&&fxRt.active?FX_RT_ROUNDTRIP_SAMPLES+(fxRt.latencyTotal||0):0;try{bus.pdcDelay.delayTime.setTargetAtTime(l/(ctx.sampleRate||44100),ctx.currentTime,0.05);}catch(e){}}// ── Realtime VST FX per-track (SHM bridge) — mọi luồng track qua bridge ───── // Mỗi track: session riêng (1 WS/SHM) cho TỪNG đường chain có âm — audio-clips // (path 'audio', splice fxEntry→mods→worklet→fxLegacyIn) và MIDI/SF (path 'sf', // splice sfEntry→mods→worklet→sfOut). Track vừa clips vừa MIDI → 2 session @@ -1627,7 +1634,9 @@ fxRtTrackReconcile(trackId,node,track);}catch(e){console.warn('rebuildTrackFxGra // Cần activeTrackNodesRef/isPlayingRef. Session giống master (fxRtStart): // fetch start → AudioWorkletNode 'sf-fx-realtime' → WS; WS open → splice // worklet vào chain hiện tại (đọc mods MỚI NHẤT — an toàn với rebuild). -const fxRtTrackStart=async(trackId,track,path)=>{const key=trackId+':'+path;const chain=fxRtTrackChain(track);if(!chain.length)return;if(fxRtTracks[key]){if(!fxRtTracks[key].node||fxRtTracks[key].node.context!==getAudioContext())fxRtTrackStop(trackId,path,true);else return;}const ctx=getAudioContext();if(!ctx||!ctx.audioWorklet)return;const chainPayload=chain.map(s=>s.type==='vst3'?{type:'vst3',path:s.path,name:s.name||s.path,preset_b64:s.preset_b64||'',bypass:false}:{type:'builtin',id:s.type,params:s.params&&Object.keys(s.params).length?s.params:{},bypass:false});let st;try{st=await window.SonicAPI.fxRealtimeStart({fx_chain:chainPayload,sample_rate:ctx.sampleRate||44100});}catch(e){console.warn('[FX RT track] start fail',trackId,path,e);return;}const entry={sessionId:st.session_id,ws:null,node:null,ready:false,path,spliced:null,chainKey:fxRtTrackChainKey(chain),paramsBySlot:chain.map(s=>fxRtEntryParams(s,null)),inQueue:[],pumpTimer:null};fxRtTracks[key]=entry;fxRtGuiPushShm();try{await ctx.audioWorklet.addModule(`${API_BASE_URL}/static/js/sf-fx-realtime.js?v=202608212200`);const node=new AudioWorkletNode(ctx,'sf-fx-realtime',{numberOfInputs:1,numberOfOutputs:1,outputChannelCount:[2]});entry.node=node;const u=new URL(API_BASE_URL);const wsProto=u.protocol==='https:'?'wss://':'ws://';const tok=localStorage.getItem('sonic_token')||'';const ws=new WebSocket(`${wsProto}${u.host}${st.ws_url}${tok?'?token='+encodeURIComponent(tok):''}`);entry.ws=ws;ws.binaryType='arraybuffer';ws.onopen=()=>{entry.ready=true;fxRtTrackSplice(trackId,path);try{(entry.paramsBySlot||[]).forEach((pl,slot)=>pl.forEach(p=>ws.send(JSON.stringify({cmd:'set_param',slot,key:p.key,value:p.value}))));}catch(e){}};ws.onmessage=ev=>{if(!(ev.data instanceof ArrayBuffer)||!entry.node)return;const f=new Float32Array(ev.data);const half=f.length/2;const l=new Float32Array(half);const r=new Float32Array(half);for(let i=0;ifxRtTrackStop(trackId,path,true);ws.onerror=()=>{try{ws.close();}catch(e){}};entry.node.port.onmessage=ev=>{const d=ev.data;if(!d||!entry.ws)return;if(d.type==='set_param'){try{entry.ws.send(JSON.stringify({cmd:'set_param',slot:d.slot,key:d.key,value:d.value}));}catch(e){}return;}if(d.type!=='in')return;if(entry.inQueue.length>64)entry.inQueue.shift();entry.inQueue.push(d.data);};const blockMs=Math.max(4,Math.round(256/(ctx.sampleRate||44100)*1000));entry.pumpTimer=setInterval(()=>{const ws=entry.ws;if(!ws||ws.readyState!==1||!entry.inQueue.length)return;const d=entry.inQueue.shift();try{ws.send(d);}catch(e){entry.inQueue.unshift(d);}},blockMs);}catch(e){console.warn('[FX RT track] setup fail',trackId,path,e);fxRtTrackStop(trackId,path,true);}};const fxRtTrackUnsplice=(trackId,path)=>{const entry=fxRtTracks[trackId+':'+path];if(!entry||!entry.spliced||!entry.node)return;try{entry.node.disconnect();try{entry.spliced.tail.connect(entry.spliced.dest);}catch(e){}entry.spliced=null;}catch(e){console.warn('[FX RT track] unsplice fail',trackId,path,e);}};const fxRtTrackSplice=(trackId,path)=>{const key=trackId+':'+path;const entry=fxRtTracks[key];const node=activeTrackNodesRef.current[trackId];if(!entry||!entry.ready||!entry.node||!node||entry.spliced)return;const tail=path==='audio'?node.fxEntry:node.sfEntry;const dest=path==='audio'?node.fxLegacyIn:node.sfOut;if(!tail||!dest)return;try{tail.disconnect(dest);entry.node.connect(dest);tail.connect(entry.node);entry.spliced={tail,dest};console.log('[FX RT track] spliced',trackId,path);}catch(e){console.warn('[FX RT track] splice fail',trackId,path,e);}};const fxRtTrackStop=(trackId,path,silent)=>{const key=trackId+':'+path;const entry=fxRtTracks[key];if(!entry)return;if(entry.pumpTimer){clearInterval(entry.pumpTimer);entry.pumpTimer=null;}entry.inQueue=[];try{fxRtTrackUnsplice(trackId,path);if(entry.node){entry.node.port.onmessage=null;try{entry.node.disconnect();}catch(e){}entry.node=null;}}catch(e){}try{if(entry.ws){entry.ws.onclose=null;entry.ws.onmessage=null;try{entry.ws.close();}catch(e){}entry.ws=null;}}catch(e){}const sid=entry.sessionId;delete fxRtTracks[key];fxRtGuiPushShm();if(sid){try{window.SonicAPI.fxRealtimeStop({session_id:sid}).catch(()=>{});}catch(e){}}};const fxRtTrackReconcile=(trackId,node,track)=>{try{const chain=fxRtTrackChain(track);const chainKey=fxRtTrackChainKey(chain);const wantPaths=chain.length?fxRtTrackPaths(track):[];const prefix=trackId+':';Object.keys(fxRtTracks).forEach(k=>{if(!k.startsWith(prefix))return;const p=k.slice(prefix.length);if(wantPaths.indexOf(p)<0)fxRtTrackStop(trackId,p,true);});if(!isPlayingRef.current)return;wantPaths.forEach(p=>{const key=trackId+':'+p;const entry=fxRtTracks[key];if(!entry)fxRtTrackStart(trackId,track,p);else if(entry.chainKey!==chainKey){fxRtTrackStop(trackId,p,true);fxRtTrackStart(trackId,track,p);}else if(entry.ready&&node){fxRtTrackUnsplice(trackId,p);// chain mới (sau rebuild) — splice lại đúng mods +const fxRtTrackStart=async(trackId,track,path)=>{const key=trackId+':'+path;const chain=fxRtTrackChain(track);if(!chain.length)return;if(fxRtTracks[key]){if(!fxRtTracks[key].node||fxRtTracks[key].node.context!==getAudioContext())fxRtTrackStop(trackId,path,true);else return;}const ctx=getAudioContext();if(!ctx||!ctx.audioWorklet)return;const chainPayload=chain.map(s=>s.type==='vst3'?{type:'vst3',path:s.path,name:s.name||s.path,preset_b64:s.preset_b64||'',bypass:false}:{type:'builtin',id:s.type,params:s.params&&Object.keys(s.params).length?s.params:{},bypass:false});let st;try{st=await window.SonicAPI.fxRealtimeStart({fx_chain:chainPayload,sample_rate:ctx.sampleRate||44100});}catch(e){console.warn('[FX RT track] start fail',trackId,path,e);return;}const entry={sessionId:st.session_id,ws:null,node:null,ready:false,path,spliced:null,chainKey:fxRtTrackChainKey(chain),paramsBySlot:chain.map(s=>fxRtEntryParams(s,null)),inQueue:[],pumpTimer:null};fxRtTracks[key]=entry;fxRtGuiPushShm();try{await ctx.audioWorklet.addModule(`${API_BASE_URL}/static/js/sf-fx-realtime.js?v=202608212300`);const node=new AudioWorkletNode(ctx,'sf-fx-realtime',{numberOfInputs:1,numberOfOutputs:1,outputChannelCount:[2]});entry.node=node;const u=new URL(API_BASE_URL);const wsProto=u.protocol==='https:'?'wss://':'ws://';const tok=localStorage.getItem('sonic_token')||'';const ws=new WebSocket(`${wsProto}${u.host}${st.ws_url}${tok?'?token='+encodeURIComponent(tok):''}`);entry.ws=ws;ws.binaryType='arraybuffer';ws.onopen=()=>{entry.ready=true;entry.inQueue=[];// bỏ audio stale trước khi WS open — hết lag khởi đầu +fxRtTrackSplice(trackId,path);try{(entry.paramsBySlot||[]).forEach((pl,slot)=>pl.forEach(p=>ws.send(JSON.stringify({cmd:'set_param',slot,key:p.key,value:p.value}))));}catch(e){}};ws.onmessage=ev=>{if(!(ev.data instanceof ArrayBuffer)||!entry.node)return;const f=new Float32Array(ev.data);const half=f.length/2;const l=new Float32Array(half);const r=new Float32Array(half);for(let i=0;ifxRtTrackStop(trackId,path,true);ws.onerror=()=>{try{ws.close();}catch(e){}};entry.node.port.onmessage=ev=>{const d=ev.data;if(!d||!entry.ws)return;if(d.type==='set_param'){try{entry.ws.send(JSON.stringify({cmd:'set_param',slot:d.slot,key:d.key,value:d.value}));}catch(e){}return;}if(d.type!=='in')return;// Burst 4 block (1024 mẫu) → tách 4 WS frame 512 floats → bridge batch 4. +if(ws.readyState===1){try{const f=new Float32Array(d.data);if(f.length>512){for(let i=0;i{const entry=fxRtTracks[trackId+':'+path];if(!entry||!entry.spliced||!entry.node)return;try{entry.node.disconnect();try{entry.spliced.tail.connect(entry.spliced.dest);}catch(e){}entry.spliced=null;}catch(e){console.warn('[FX RT track] unsplice fail',trackId,path,e);}};const fxRtTrackSplice=(trackId,path)=>{const key=trackId+':'+path;const entry=fxRtTracks[key];const node=activeTrackNodesRef.current[trackId];if(!entry||!entry.ready||!entry.node||!node||entry.spliced)return;const tail=path==='audio'?node.fxEntry:node.sfEntry;const dest=path==='audio'?node.fxLegacyIn:node.sfOut;if(!tail||!dest)return;try{tail.disconnect(dest);entry.node.connect(dest);tail.connect(entry.node);entry.spliced={tail,dest};console.log('[FX RT track] spliced',trackId,path);}catch(e){console.warn('[FX RT track] splice fail',trackId,path,e);}};const fxRtTrackStop=(trackId,path,silent)=>{const key=trackId+':'+path;const entry=fxRtTracks[key];if(!entry)return;if(entry.pumpTimer){clearInterval(entry.pumpTimer);entry.pumpTimer=null;}entry.inQueue=[];try{fxRtTrackUnsplice(trackId,path);if(entry.node){entry.node.port.onmessage=null;try{entry.node.disconnect();}catch(e){}entry.node=null;}}catch(e){}try{if(entry.ws){entry.ws.onclose=null;entry.ws.onmessage=null;try{entry.ws.close();}catch(e){}entry.ws=null;}}catch(e){}const sid=entry.sessionId;delete fxRtTracks[key];fxRtGuiPushShm();if(sid){try{window.SonicAPI.fxRealtimeStop({session_id:sid}).catch(()=>{});}catch(e){}}};const fxRtTrackReconcile=(trackId,node,track)=>{try{const chain=fxRtTrackChain(track);const chainKey=fxRtTrackChainKey(chain);const wantPaths=chain.length?fxRtTrackPaths(track):[];const prefix=trackId+':';Object.keys(fxRtTracks).forEach(k=>{if(!k.startsWith(prefix))return;const p=k.slice(prefix.length);if(wantPaths.indexOf(p)<0)fxRtTrackStop(trackId,p,true);});if(!isPlayingRef.current)return;wantPaths.forEach(p=>{const key=trackId+':'+p;const entry=fxRtTracks[key];if(!entry)fxRtTrackStart(trackId,track,p);else if(entry.chainKey!==chainKey){fxRtTrackStop(trackId,p,true);fxRtTrackStart(trackId,track,p);}else if(entry.ready&&node){fxRtTrackUnsplice(trackId,p);// chain mới (sau rebuild) — splice lại đúng mods fxRtTrackSplice(trackId,p);}});}catch(e){console.warn('fxRtTrackReconcile error:',e);}};const fxRtTrackStopAll=()=>{Object.keys(fxRtTracks).forEach(k=>{const i=k.lastIndexOf(':');fxRtTrackStop(k.slice(0,i),k.slice(i+1),true);});};// Route the SHARED FluidSynth output through the gainNode of the single // audible MIDI track so its FX Rack chain / fader / pan affect the soundfont // instrument (unified_fx_rack_panel.md + user request). With more than one diff --git a/app/static/js/sf-fx-realtime.js b/app/static/js/sf-fx-realtime.js index f402248..dbe17f3 100644 --- a/app/static/js/sf-fx-realtime.js +++ b/app/static/js/sf-fx-realtime.js @@ -4,6 +4,13 @@ // nhận block đã xử lý từ main thread → queue → phát ra output. Queue rỗng → // zero fill (WS chưa kịp round-trip). // +// BURST (fix lag/giật/crackle): gom 4 block (1024 mẫu) vào 1 frame rồi post. +// Bridge VST3 (fx_vst_bridge) chạy chậm hơn realtime khi take=1 (256 mẫu: +// ~5.5ms/call > budget 5.33ms → 179/s < 187.5/s → engine drop input → +// crackle). Gửi burst 4 block → bridge take=4 → 1 call process 1024 mẫu +// (amortize overhead: 14-19ms/4 block ≈ 3.5-4.75ms/block) → đủ 187.5/s. +// Latency thêm ~4 block (~21ms) — worklet outQueue 32 block hấp thụ. +// // Latency ~2 block; backlog > 32 block (~190ms) → drop block cũ nhất. Cap cao // hấp thụ jitter main thread (rAF renderFrame nặng khi mở MASTERING PANEL) mà // không drop. ponytail: thay drop bằng adaptive stretch khi cần chất lượng cao. @@ -11,8 +18,10 @@ class SfFxRealtimeProcessor extends AudioWorkletProcessor { constructor() { super(); this.BLOCK = 256; // FXRT_BLOCK - this.inL = new Float32Array(this.BLOCK); - this.inR = new Float32Array(this.BLOCK); + this.BURST = 4; // số block gom 1 frame (bridge batch) + this.bufLen = this.BLOCK * this.BURST; + this.inL = new Float32Array(this.bufLen); + this.inR = new Float32Array(this.bufLen); this.inCount = 0; this.outQueue = []; // [{l: Float32Array, r: Float32Array}] this.outPos = 0; @@ -31,19 +40,20 @@ class SfFxRealtimeProcessor extends AudioWorkletProcessor { const outL = output[0]; const outR = output[1]; - // Góp input thành block 256 → gửi main thread (transferable buffer). + // Góp input thành burst 4 block (1024 mẫu) → gửi main thread + // (transferable buffer). Main thread tách 4 WS frame → bridge batch 4. if (input && input[0]) { const inL = input[0]; const inR = input[1] || input[0]; for (let i = 0; i < n; i++) { - if (this.inCount < this.BLOCK) { + if (this.inCount < this.bufLen) { this.inL[this.inCount] = inL[i]; this.inR[this.inCount] = inR[i]; } this.inCount++; - if (this.inCount >= this.BLOCK) { - const inter = new Float32Array(this.BLOCK * 2); - for (let j = 0; j < this.BLOCK; j++) { + if (this.inCount >= this.bufLen) { + const inter = new Float32Array(this.bufLen * 2); + for (let j = 0; j < this.bufLen; j++) { inter[j * 2] = this.inL[j]; inter[j * 2 + 1] = this.inR[j]; } diff --git a/app/templates/index.html b/app/templates/index.html index 10377f9..21d7fd4 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -51,7 +51,7 @@ - +