3dtours/RecipesCam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(heal): restore direct single-threaded WASM inpainting on mobile, remove worker to prevent WebKit reload

Browse Source
This commit is contained in:
3dtours
2026-10-16 11:32:06 +07:00
parent b90086c8a8
commit d579a67ed9
3 changed files with 45 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/frontend/src/engine/inpaint.ts
+44 -121
View File
@@ -9,11 +9,10 @@ import type * as Ort from 'onnxruntime-web';
// `migan_pipeline_v2.onnx` weights inpaint-web ships) is 27MB and runs through
// onnxruntime-web:
// 1. WebGPU on desktop browsers with hardware acceleration (ultra fast ~50ms).
// 2. Dedicated Web Worker executing single-threaded WASM with zero-copy transferable
// buffers on mobile devices (iOS WebKit and Android) and non-WebGPU desktops.
// Running off the main thread with bounded memory guarantees 100% crash-free
// neural inference without WebKit GPUProcess crashes, watchdog timeouts, or Jetsam kills.
// 3. Mathematical anisotropic isophote synthesis as a robust instant fallback.
// 2. Direct single-threaded WASM execution on mobile devices (iOS WebKit and Android)
// with proxy=false, numThreads=1, enableCpuMemArena=false, and bounded 256px resolution.
// Runs rock-solid in-thread without Web Worker memory limits or WebKit GPUProcess crashes.
// 3. Mathematical anisotropic isophote synthesis as an instant fallback.
const MODEL_URL = '/models/migan_pipeline_v2.onnx';
const WASM_DIR = '/wasm/ort/';
@@ -98,9 +97,13 @@ function load(): Promise<Loaded> {
loaded = (async () => {
const ort = await import('onnxruntime-web');
ort.env.wasm.wasmPaths = WASM_DIR;
ort.env.wasm.numThreads = self.crossOriginIsolated
const mobile = isMobile();
ort.env.wasm.numThreads = mobile
? 1
: self.crossOriginIsolated
? Math.min(4, navigator.hardwareConcurrency || 1)
: 1;
ort.env.wasm.proxy = false;
ort.env.webgpu.powerPreference = 'high-performance';
const session = await openSession(ort);
return { ort, session };
@@ -112,65 +115,11 @@ function load(): Promise<Loaded> {
return loaded;
}
// Dedicated Web Worker execution for WASM on desktop
let inpaintWorker: Worker | null = null;
let workerMsgId = 0;
const workerCallbacks = new Map<number, (patch: Uint8Array | null) => void>();
function getInpaintWorker(): Worker | null {
if (inpaintWorker) return inpaintWorker;
if (typeof Worker === 'undefined') return null;
try {
const w = new Worker(new URL('./inpaintWorker.ts', import.meta.url), { type: 'module' });
w.onmessage = (e: MessageEvent<{ id: number; patch?: Uint8Array; error?: string }>) => {
const cb = workerCallbacks.get(e.data.id);
if (cb) {
workerCallbacks.delete(e.data.id);
cb(e.data.patch ?? null);
}
};
w.onerror = (err) => {
console.warn('[inpaint worker error]', err);
for (const cb of workerCallbacks.values()) cb(null);
workerCallbacks.clear();
w.terminate();
if (inpaintWorker === w) inpaintWorker = null;
};
inpaintWorker = w;
} catch (err) {
console.warn('[inpaint] worker instantiation failed:', err);
return null;
}
return inpaintWorker;
}
function runInpaintWorker(src: Uint8Array, mask: Uint8Array, w: number, h: number): Promise<Uint8Array | null> {
const worker = getInpaintWorker();
if (!worker) return Promise.resolve(null);
const id = ++workerMsgId;
return new Promise((resolve) => {
const timer = setTimeout(() => {
workerCallbacks.delete(id);
resolve(null);
}, 20000);
workerCallbacks.set(id, (patch) => {
clearTimeout(timer);
resolve(patch);
});
try {
worker.postMessage({ id, src, mask, w, h }, [src.buffer, mask.buffer]);
} catch {
worker.postMessage({ id, src, mask, w, h });
}
});
}
// Start fetching the model before the brush that needs it.
export function preloadInpaint(): void {
for (const file of [MODEL_URL, ...RUNTIME_FILES.map((f) => WASM_DIR + f), WASM_DIR + 'ort-wasm-simd-threaded.wasm']) {
void fetch(file).catch(() => {});
}
void getInpaintWorker();
}
/**
@@ -417,12 +366,6 @@ export function clearInpaintCache(): void {
export async function releaseInpaint(): Promise<void> {
patchCache.clear();
if (inpaintWorker) {
for (const cb of workerCallbacks.values()) cb(null);
workerCallbacks.clear();
inpaintWorker.terminate();
inpaintWorker = null;
}
const pending = loaded;
loaded = null;
if (!pending) return;
@@ -485,65 +428,45 @@ export async function inpaintCrop(
const runPromise = (async (): Promise<Uint8Array | null> => {
const { src, hole: mask } = cropPlate(rgba, width, height, hole, w, h);
// 1. Desktop: Try WebGPU hardware acceleration first if supported (ultra fast ~50ms)
if (await hasDevice()) {
let in0: Ort.Tensor | null = null;
let in1: Ort.Tensor | null = null;
let out: Record<string, Ort.Tensor> | null = null;
try {
const { ort, session } = await load();
in0 = new ort.Tensor('uint8', src, [1, 3, h, w]);
in1 = new ort.Tensor('uint8', mask, [1, 1, h, w]);
out = await session.run({
[session.inputNames[0]]: in0,
[session.inputNames[1]]: in1,
});
const tensor = out[session.outputNames[0]];
const oplane = w * h;
const data = tensor.data as Uint8Array;
const patch = new Uint8Array(4 * w * h);
for (let i = 0; i < oplane; i++) {
patch[i * 4] = data[i];
patch[i * 4 + 1] = data[oplane + i];
patch[i * 4 + 2] = data[2 * oplane + i];
patch[i * 4 + 3] = 255;
}
if (patchCache.size > 50) {
const first = patchCache.keys().next().value;
if (first) patchCache.delete(first);
}
patchCache.set(key, patch.slice());
return patch;
} catch (err) {
console.warn('[inpaint] WebGPU inference failed, attempting worker:', err);
} finally {
try {
in0?.dispose?.();
in1?.dispose?.();
if (out) {
for (const k in out) out[k]?.dispose?.();
}
} catch {}
}
}
// 2. Desktop: Run in Dedicated Web Worker (WASM off main thread)
let in0: Ort.Tensor | null = null;
let in1: Ort.Tensor | null = null;
let out: Record<string, Ort.Tensor> | null = null;
try {
const patch = await runInpaintWorker(src, mask, w, h);
if (patch) {
if (patchCache.size > 50) {
const first = patchCache.keys().next().value;
if (first) patchCache.delete(first);
}
patchCache.set(key, patch.slice());
return patch;
const { ort, session } = await load();
in0 = new ort.Tensor('uint8', src, [1, 3, h, w]);
in1 = new ort.Tensor('uint8', mask, [1, 1, h, w]);
out = await session.run({
[session.inputNames[0]]: in0,
[session.inputNames[1]]: in1,
});
const tensor = out[session.outputNames[0]];
const oplane = w * h;
const data = tensor.data as Uint8Array;
const patch = new Uint8Array(4 * w * h);
for (let i = 0; i < oplane; i++) {
patch[i * 4] = data[i];
patch[i * 4 + 1] = data[oplane + i];
patch[i * 4 + 2] = data[2 * oplane + i];
patch[i * 4 + 3] = 255;
}
if (patchCache.size > 50) {
const first = patchCache.keys().next().value;
if (first) patchCache.delete(first);
}
patchCache.set(key, patch.slice());
return patch;
} catch (err) {
console.warn('[inpaint] Worker execution error:', err);
console.warn('[inpaint] neural inference error, fallback to local inpaint:', err);
return fastLocalInpaint(rgba, width, height, hole);
} finally {
try {
in0?.dispose?.();
in1?.dispose?.();
if (out) {
for (const k in out) out[k]?.dispose?.();
}
} catch {}
}
// 3. High quality local inpaint fallback
return fastLocalInpaint(rgba, width, height, hole);
})();
inFlightInpaint = runPromise;
docker/frontend/src/engine/inpaintWorker.ts
-83
View File
@@ -1,83 +0,0 @@
import * as ort from 'onnxruntime-web';
const MODEL_URL = '/models/migan_pipeline_v2.onnx';
const WASM_DIR = '/wasm/ort/';
export interface InpaintAsk {
id: number;
src: Uint8Array;
mask: Uint8Array;
w: number;
h: number;
}
export interface InpaintReply {
id: number;
patch?: Uint8Array;
error?: string;
}
let loaded: Promise<ort.InferenceSession> | null = null;
function loadSession(): Promise<ort.InferenceSession> {
if (!loaded) {
loaded = (async () => {
ort.env.wasm.wasmPaths = WASM_DIR;
ort.env.wasm.numThreads = 1;
ort.env.wasm.proxy = false;
return await ort.InferenceSession.create(MODEL_URL, {
executionProviders: ['wasm'],
enableCpuMemArena: false,
enableMemPattern: false,
graphOptimizationLevel: 'all',
});
})().catch((err) => {
loaded = null;
throw err;
});
}
return loaded;
}
const scope = self as unknown as {
onmessage: ((event: MessageEvent<InpaintAsk>) => void) | null;
postMessage: (message: InpaintReply, transfer?: Transferable[]) => void;
};
scope.onmessage = async (event: MessageEvent<InpaintAsk>) => {
const { id, src, mask, w, h } = event.data;
let in0: ort.Tensor | null = null;
let in1: ort.Tensor | null = null;
let out: Record<string, ort.Tensor> | null = null;
try {
const session = await loadSession();
in0 = new ort.Tensor('uint8', src, [1, 3, h, w]);
in1 = new ort.Tensor('uint8', mask, [1, 1, h, w]);
out = await session.run({
[session.inputNames[0]]: in0,
[session.inputNames[1]]: in1,
});
const tensor = out[session.outputNames[0]];
const data = tensor.data as Uint8Array;
const oplane = w * h;
const patch = new Uint8Array(4 * w * h);
for (let i = 0; i < oplane; i++) {
patch[i * 4] = data[i];
patch[i * 4 + 1] = data[oplane + i];
patch[i * 4 + 2] = data[2 * oplane + i];
patch[i * 4 + 3] = 255;
}
scope.postMessage({ id, patch }, [patch.buffer]);
} catch (err: unknown) {
scope.postMessage({ id, error: err instanceof Error ? err.message : String(err) });
} finally {
try {
in0?.dispose?.();
in1?.dispose?.();
if (out) {
for (const k in out) out[k]?.dispose?.();
}
} catch {}
}
};
docker/recipescam-images.tar.gz.sha256
+1 -1
View File
@@ -1 +1 @@
4070d31abbd26ac714d4b0ef8e335e7db53f2ba5e5025f689df5e034bc2e7099 recipescam-images.tar.gz
3c7123f3dcbd7fc20246a4e6d7f1b5c7eebb53f3dacf4ff557b992491bf7006a recipescam-images.tar.gz