G3.1: dedicated bridge heartbeat — watchdog liveness independent of audio loop
This commit is contained in:
+63
-30
@@ -374,13 +374,19 @@ pub fn run() {
|
||||
let _ = spawn_bridge(&app_handle, &res_dir, &exe_dir, &mut log_line, &spawn_log_path);
|
||||
|
||||
// ── Audio pump: bridge SHM -> WebView `bridge-audio` events ──
|
||||
// B10 health: if bridgeWriteIndex stalls for 3s the bridge is dead —
|
||||
// respawn once, then emit `bridge-down` so the UI falls back to WASM.
|
||||
// G3.1: bridge liveness keys on the DEDICATED heartbeat (a bridge
|
||||
// thread, independent of the audio loop). A slow-but-alive audio
|
||||
// loop no longer looks dead. writeIndex is now only for audio
|
||||
// emission; a stalled writeIndex with a live heartbeat is logged
|
||||
// as an audio stall, never a restart.
|
||||
let pump_handle = app.handle().clone();
|
||||
std::thread::spawn(move || {
|
||||
let mut last_index: u32 = 0;
|
||||
let mut last_change = std::time::Instant::now();
|
||||
let mut last_hb: u32 = 0;
|
||||
let mut last_hb_change = std::time::Instant::now();
|
||||
let mut last_audio_change = std::time::Instant::now();
|
||||
let mut down_emitted = false;
|
||||
let mut stall_logged = false;
|
||||
let mut restart_attempts = 0u32;
|
||||
loop {
|
||||
// App đang tắt (window Destroyed) — dừng pump, không respawn bridge.
|
||||
@@ -395,7 +401,25 @@ pub fn run() {
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let advanced = guard.as_ref().map(|shm| {
|
||||
// Liveness = heartbeat advancing (~10Hz from bridge thread).
|
||||
let hb_advanced = guard.as_ref().map(|shm| {
|
||||
let hb = shm.heartbeat();
|
||||
if hb != last_hb {
|
||||
last_hb = hb;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
});
|
||||
if hb_advanced == Some(true) {
|
||||
last_hb_change = std::time::Instant::now();
|
||||
down_emitted = false;
|
||||
// Bridge sống lại sau restart — đóng "sự cố" để lần
|
||||
// stall sau lại được hưởng 1 restart (mỗi incident 1 lần).
|
||||
restart_attempts = 0;
|
||||
}
|
||||
// Audio emission = writeIndex advancing (audio loop alive).
|
||||
let audio_advanced = guard.as_ref().map(|shm| {
|
||||
let idx = shm.write_index();
|
||||
if idx != last_index {
|
||||
last_index = idx;
|
||||
@@ -404,33 +428,41 @@ pub fn run() {
|
||||
false
|
||||
}
|
||||
});
|
||||
// G0.3: an openGUI attach job (reload/attachView on the
|
||||
// bridge worker) can legitimately outlast the 3s stall
|
||||
// deadline if it blocks the audio loop's message pump —
|
||||
// do not kill the bridge for that. Real stalls are still
|
||||
// caught: attach never takes > 15s; a dead bridge leaves
|
||||
// the flag stuck and is restarted after the longer wait.
|
||||
let attach_deadline = guard
|
||||
.as_ref()
|
||||
.map(|s| s.attach_in_flight())
|
||||
.unwrap_or(false);
|
||||
let stall_deadline = if attach_deadline {
|
||||
std::time::Duration::from_secs(15)
|
||||
} else {
|
||||
std::time::Duration::from_secs(3)
|
||||
};
|
||||
if advanced == Some(true) {
|
||||
last_change = std::time::Instant::now();
|
||||
down_emitted = false;
|
||||
// Bridge đã sống lại sau restart — đóng "sự cố" để lần
|
||||
// stall sau lại được hưởng 1 restart (mỗi incident 1 lần).
|
||||
restart_attempts = 0;
|
||||
if audio_advanced == Some(true) {
|
||||
last_audio_change = std::time::Instant::now();
|
||||
stall_logged = false;
|
||||
let (l, r) = guard.as_ref().map(|s| s.read_audio()).unwrap_or(([0f32; shm::AUDIO_BLOCK_SIZE], [0f32; shm::AUDIO_BLOCK_SIZE]));
|
||||
let _ = pump_handle.emit(
|
||||
"bridge-audio",
|
||||
AudioFrame { l: l.to_vec(), r: r.to_vec() },
|
||||
);
|
||||
} else if last_change.elapsed() >= stall_deadline && !down_emitted {
|
||||
}
|
||||
// G3.1 FIX: nhả guard TRƯỚC stall log / watchdog kill+spawn.
|
||||
// kill_bridge -> flush_bridge_state khóa chính ShmState mutex
|
||||
// mà pump đang giữ -> deadlock khi watchdog trigger (không
|
||||
// restart, không log, bridge chết vĩnh viễn).
|
||||
drop(guard);
|
||||
if audio_advanced != Some(true)
|
||||
&& hb_advanced == Some(true)
|
||||
&& last_audio_change.elapsed() >= std::time::Duration::from_secs(3)
|
||||
&& !stall_logged
|
||||
{
|
||||
// Heartbeat OK nhưng audio loop không push block — stall
|
||||
// thực sự (không phải bridge chết). Log, không restart.
|
||||
stall_logged = true;
|
||||
let log_dir = std::env::var("APPDATA").unwrap_or_else(|_| ".".into());
|
||||
let bridge_log = std::path::Path::new(&log_dir)
|
||||
.join("SonicForgeDAW").join("logs").join("bridge.log");
|
||||
if let Ok(mut f) = std::fs::OpenOptions::new()
|
||||
.create(true).append(true).open(&bridge_log)
|
||||
{
|
||||
use std::io::Write;
|
||||
let _ = f.write_all(b"[tauri] audio stall (heartbeat OK)\n");
|
||||
}
|
||||
}
|
||||
if hb_advanced == Some(false) && !down_emitted
|
||||
&& last_hb_change.elapsed() >= std::time::Duration::from_secs(3)
|
||||
{
|
||||
// Check lai truoc khi respawn — Destroyed co the vua set flag.
|
||||
if SHUTTING_DOWN.load(Ordering::Relaxed) {
|
||||
break;
|
||||
@@ -446,7 +478,8 @@ pub fn run() {
|
||||
let log_dir = std::env::var("APPDATA").unwrap_or_else(|_| ".".into());
|
||||
let log_path = std::path::Path::new(&log_dir)
|
||||
.join("SonicForgeDAW").join("logs").join("spawn.log");
|
||||
let mut line = String::from("[tauri] bridge stalled 3s — restart attempt #1\n");
|
||||
let mut line = String::from("[tauri] bridge heartbeat stalled 3s — restart attempt #1
|
||||
");
|
||||
// FIX: kill bridge cũ trước — không được để 2 bridge
|
||||
// cùng map SHM (race control queue / double load).
|
||||
kill_bridge(&pump_handle);
|
||||
@@ -458,9 +491,10 @@ pub fn run() {
|
||||
.create(true).append(true).open(&bridge_log)
|
||||
{
|
||||
use std::io::Write;
|
||||
let _ = f.write_all(b"[tauri] bridge restarted (attempt 1)\n");
|
||||
let _ = f.write_all(b"[tauri] bridge restarted (attempt 1)
|
||||
");
|
||||
}
|
||||
last_change = std::time::Instant::now();
|
||||
last_hb_change = std::time::Instant::now();
|
||||
down_emitted = false;
|
||||
} else {
|
||||
let _ = pump_handle.emit("bridge-down", ());
|
||||
@@ -469,7 +503,6 @@ pub fn run() {
|
||||
let _ = pump_handle.emit("bridge-down", ());
|
||||
}
|
||||
}
|
||||
drop(guard);
|
||||
std::thread::sleep(std::time::Duration::from_millis(1));
|
||||
}
|
||||
});
|
||||
|
||||
@@ -63,6 +63,10 @@ pub struct SharedAudioBufferIPC {
|
||||
// Append-only field — offsets of control_queue (2836) and
|
||||
// control_queue_count (11156) are unchanged.
|
||||
pub attach_in_flight: u32,
|
||||
// G3.1: dedicated heartbeat — incremented ~10Hz by a bridge thread
|
||||
// independent of the audio loop. Watchdog liveness keys on this.
|
||||
// Append-only field: offsets above are unchanged.
|
||||
pub heartbeat: u32,
|
||||
}
|
||||
|
||||
pub struct Shm {
|
||||
@@ -166,6 +170,10 @@ impl Shm {
|
||||
pub fn attach_in_flight(&self) -> bool {
|
||||
self.ipc().attach_in_flight != 0
|
||||
}
|
||||
|
||||
pub fn heartbeat(&self) -> u32 {
|
||||
self.ipc().heartbeat
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Shm {
|
||||
@@ -188,6 +196,6 @@ mod tests {
|
||||
// native_bridge/tests/shm_selfcheck.cpp asserts the same sizes.
|
||||
assert_eq!(std::mem::size_of::<MidiEventIPC>(), 12);
|
||||
assert_eq!(std::mem::size_of::<ControlEventIPC>(), 1040);
|
||||
assert_eq!(std::mem::size_of::<SharedAudioBufferIPC>(), 11164);
|
||||
assert_eq!(std::mem::size_of::<SharedAudioBufferIPC>(), 11168);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user