fix(heal): run neural inpaint via dedicated worker on mobile to eliminate blur fallback and prevent browser reload
This commit is contained in:
@@ -5,17 +5,15 @@ import type * as Ort from 'onnxruntime-web';
|
||||
// right for a speck of dust on a wall and thin over anything with structure: a
|
||||
// hole across a shirt comes back as the average of a shirt.
|
||||
//
|
||||
// The fill is now a learned one on desktop, and an anisotropic gradient & texture
|
||||
// synthesizer on mobile. MI-GAN (andraniksargsyan/migan, the
|
||||
// The fill is a learned one. MI-GAN (andraniksargsyan/migan, the
|
||||
// `migan_pipeline_v2.onnx` weights inpaint-web ships) is 27MB and runs through
|
||||
// onnxruntime-web on desktop machines with dedicated GPU/CPU memory pools.
|
||||
//
|
||||
// On mobile devices (iOS WebKit and Android), WebKit imposes strict memory ceilings
|
||||
// (~300MB Jetsam limit) and experimental WebGPU implementations that crash WebKit's
|
||||
// GPUProcess, causing browsers to reload or crash. On mobile, fastLocalInpaint runs
|
||||
// continuous gradient-guided isophote inpainting with Poisson boundary relaxation in <25ms,
|
||||
// guaranteeing 100% crash-free operation without blocky mosaic tiles ("tạo bệt")
|
||||
// and without flat grey smears ("mảng xám").
|
||||
// onnxruntime-web:
|
||||
// 1. WebGPU on desktop browsers with hardware acceleration (ultra fast ~50ms).
|
||||
// 2. Dedicated Web Worker executing single-threaded WASM with zero-copy transferable
|
||||
// buffers on mobile devices (iOS WebKit and Android) and non-WebGPU desktops.
|
||||
// Running off the main thread with bounded memory guarantees 100% crash-free
|
||||
// neural inference without WebKit GPUProcess crashes, watchdog timeouts, or Jetsam kills.
|
||||
// 3. Mathematical anisotropic isophote synthesis as a robust instant fallback.
|
||||
|
||||
const MODEL_URL = '/models/migan_pipeline_v2.onnx';
|
||||
const WASM_DIR = '/wasm/ort/';
|
||||
@@ -120,7 +118,6 @@ let workerMsgId = 0;
|
||||
const workerCallbacks = new Map<number, (patch: Uint8Array | null) => void>();
|
||||
|
||||
function getInpaintWorker(): Worker | null {
|
||||
if (isMobile()) return null;
|
||||
if (inpaintWorker) return inpaintWorker;
|
||||
if (typeof Worker === 'undefined') return null;
|
||||
try {
|
||||
@@ -160,14 +157,16 @@ function runInpaintWorker(src: Uint8Array, mask: Uint8Array, w: number, h: numbe
|
||||
clearTimeout(timer);
|
||||
resolve(patch);
|
||||
});
|
||||
worker.postMessage({ id, src, mask, w, h });
|
||||
try {
|
||||
worker.postMessage({ id, src, mask, w, h }, [src.buffer, mask.buffer]);
|
||||
} catch {
|
||||
worker.postMessage({ id, src, mask, w, h });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Start fetching the model before the brush that needs it on desktop.
|
||||
// Skipped on mobile to preserve bandwidth, battery, and avoid tab Jetsam kills.
|
||||
// Start fetching the model before the brush that needs it.
|
||||
export function preloadInpaint(): void {
|
||||
if (isMobile()) return;
|
||||
for (const file of [MODEL_URL, ...RUNTIME_FILES.map((f) => WASM_DIR + f), WASM_DIR + 'ort-wasm-simd-threaded.wasm']) {
|
||||
void fetch(file).catch(() => {});
|
||||
}
|
||||
@@ -473,20 +472,6 @@ export async function inpaintCrop(
|
||||
return patchCache.get(key)!.slice();
|
||||
}
|
||||
|
||||
// On mobile devices (iOS WebKit and Android), executing a 27MB neural model
|
||||
// triggers WebKit GPUProcess crashes (WebGPU) or Jetsam OOM kills (WASM),
|
||||
// causing browser reload / crashes. We use fastLocalInpaint on mobile:
|
||||
// continuous anisotropic isophote synthesis in ~15ms, guaranteed crash-free.
|
||||
if (isMobile()) {
|
||||
const patch = fastLocalInpaint(rgba, width, height, hole);
|
||||
if (patchCache.size > 50) {
|
||||
const first = patchCache.keys().next().value;
|
||||
if (first) patchCache.delete(first);
|
||||
}
|
||||
patchCache.set(key, patch.slice());
|
||||
return patch;
|
||||
}
|
||||
|
||||
// Single-flight lock: serialize inference executions to prevent memory spikes
|
||||
if (inFlightInpaint) {
|
||||
try {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type * as Ort from 'onnxruntime-web';
|
||||
import * as ort from 'onnxruntime-web';
|
||||
|
||||
const MODEL_URL = '/models/migan_pipeline_v2.onnx';
|
||||
const WASM_DIR = '/wasm/ort/';
|
||||
@@ -17,23 +17,24 @@ export interface InpaintReply {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
let loaded: Promise<{ ort: typeof Ort; session: Ort.InferenceSession }> | null = null;
|
||||
let loaded: Promise<ort.InferenceSession> | null = null;
|
||||
|
||||
function loadSession(): Promise<{ ort: typeof Ort; session: Ort.InferenceSession }> {
|
||||
function loadSession(): Promise<ort.InferenceSession> {
|
||||
if (!loaded) {
|
||||
loaded = (async () => {
|
||||
const ort = await import('onnxruntime-web');
|
||||
ort.env.wasm.wasmPaths = WASM_DIR;
|
||||
ort.env.wasm.numThreads = 1;
|
||||
ort.env.wasm.proxy = false;
|
||||
const session = await ort.InferenceSession.create(MODEL_URL, {
|
||||
return await ort.InferenceSession.create(MODEL_URL, {
|
||||
executionProviders: ['wasm'],
|
||||
enableCpuMemArena: false,
|
||||
enableMemPattern: false,
|
||||
graphOptimizationLevel: 'all',
|
||||
});
|
||||
return { ort, session };
|
||||
})();
|
||||
})().catch((err) => {
|
||||
loaded = null;
|
||||
throw err;
|
||||
});
|
||||
}
|
||||
return loaded;
|
||||
}
|
||||
@@ -45,11 +46,11 @@ const scope = self as unknown as {
|
||||
|
||||
scope.onmessage = async (event: MessageEvent<InpaintAsk>) => {
|
||||
const { id, src, mask, w, h } = event.data;
|
||||
let in0: Ort.Tensor | null = null;
|
||||
let in1: Ort.Tensor | null = null;
|
||||
let out: Record<string, Ort.Tensor> | null = null;
|
||||
let in0: ort.Tensor | null = null;
|
||||
let in1: ort.Tensor | null = null;
|
||||
let out: Record<string, ort.Tensor> | null = null;
|
||||
try {
|
||||
const { ort, session } = await loadSession();
|
||||
const session = await loadSession();
|
||||
in0 = new ort.Tensor('uint8', src, [1, 3, h, w]);
|
||||
in1 = new ort.Tensor('uint8', mask, [1, 1, h, w]);
|
||||
|
||||
|
||||
@@ -1 +1 @@
|
||||
38762621392ee871ae7f528a596f98bdb5cbb12e68cda4be240053e2879e3434 recipescam-images.tar.gz
|
||||
4070d31abbd26ac714d4b0ef8e335e7db53f2ba5e5025f689df5e034bc2e7099 recipescam-images.tar.gz
|
||||
|
||||
Reference in New Issue
Block a user