diff --git a/docker/frontend/src/engine/inpaint.ts b/docker/frontend/src/engine/inpaint.ts index 8be30e0..409fa77 100644 --- a/docker/frontend/src/engine/inpaint.ts +++ b/docker/frontend/src/engine/inpaint.ts @@ -464,7 +464,8 @@ export function fastLocalInpaint( } } - // 2. Base structural layer: Dirichlet Poisson / Laplace solve (∇² I = 0) with SOR + // 2. Base structural layer: Initialize with smooth transfinite Coons / ray projection + // across the hole to eliminate Voronoi polygonal cones, then relax with SOR (∇² I = 0). const baseR = new Float32Array(total); const baseG = new Float32Array(total); const baseB = new Float32Array(total); @@ -475,11 +476,74 @@ export function fastLocalInpaint( baseR[i] = rgba[p]; baseG[i] = rgba[p + 1]; baseB[i] = rgba[p + 2]; - } else { - const donor = (ny[i] * w + nx[i]) << 2; - baseR[i] = rgba[donor]; - baseG[i] = rgba[donor + 1]; - baseB[i] = rgba[donor + 2]; + } + } + + for (let y = ymin; y <= ymax; y++) { + const row = y * w; + for (let x = xmin; x <= xmax; x++) { + const idx = row + x; + if (clean[idx]) continue; + + let xl = x; + while (xl >= 0 && !clean[row + xl]) xl--; + let xr = x; + while (xr < w && !clean[row + xr]) xr++; + + let yt = y; + while (yt >= 0 && !clean[yt * w + x]) yt--; + let yb = y; + while (yb < h && !clean[yb * w + x]) yb++; + + let sumR = 0, sumG = 0, sumB = 0, sumW = 0; + + if (xl >= 0) { + const d = x - xl; + const weight = 1.0 / (d * d); + const p = (row + xl) << 2; + sumR += rgba[p] * weight; + sumG += rgba[p + 1] * weight; + sumB += rgba[p + 2] * weight; + sumW += weight; + } + if (xr < w) { + const d = xr - x; + const weight = 1.0 / (d * d); + const p = (row + xr) << 2; + sumR += rgba[p] * weight; + sumG += rgba[p + 1] * weight; + sumB += rgba[p + 2] * weight; + sumW += weight; + } + if (yt >= 0) { + const d = y - yt; + const weight = 1.0 / (d * d); + const p = (yt * w + x) << 2; + sumR += rgba[p] * weight; + sumG += rgba[p + 1] * weight; + sumB += rgba[p + 2] * weight; + sumW += weight; + } + if (yb < h) { + const d = yb - y; + const weight = 1.0 / (d * d); + const p = (yb * w + x) << 2; + sumR += rgba[p] * weight; + sumG += rgba[p + 1] * weight; + sumB += rgba[p + 2] * weight; + sumW += weight; + } + + if (sumW > 0) { + baseR[idx] = sumR / sumW; + baseG[idx] = sumG / sumW; + baseB[idx] = sumB / sumW; + } else { + const donor = (ny[idx] * w + nx[idx]) << 2; + baseR[idx] = rgba[donor]; + baseG[idx] = rgba[donor + 1]; + baseB[idx] = rgba[donor + 2]; + } } } @@ -549,7 +613,7 @@ export function fastLocalInpaint( } } - // 4. Structure Tensor on local clean boundary band (within 12px) + // 4. Structure Tensor on local clean boundary band (clean pixels adjacent to hole) let sumGxx = 0, sumGyy = 0, sumGxy = 0, bandCount = 0; let hpVarSum = 0; @@ -558,11 +622,25 @@ export function fastLocalInpaint( for (let x = xmin; x <= xmax; x++) { const idx = row + x; if (!clean[idx]) continue; - const d = dsq[idx]; - // Clean pixels close to hole boundary - if (d > 0 && d <= 144) { + + // Check if adjacent to hole within 2 pixels + let closeToHole = false; + for (let dy = -2; dy <= 2 && !closeToHole; dy++) { + const ny = y + dy; + if (ny < 0 || ny >= h) continue; + const nrow = ny * w; + for (let dx = -2; dx <= 2; dx++) { + const nx = x + dx; + if (nx < 0 || nx >= w) continue; + if (!clean[nrow + nx]) { + closeToHole = true; + break; + } + } + } + + if (closeToHole) { const p = idx << 2; - const lum = rgba[p] * 0.299 + rgba[p + 1] * 0.587 + rgba[p + 2] * 0.114; const lumL = rgba[(row + Math.max(0, x - 1)) << 2] * 0.299 + rgba[(row + Math.max(0, x - 1)) << 2 + 1] * 0.587 + rgba[(row + Math.max(0, x - 1)) << 2 + 2] * 0.114; const lumR = rgba[(row + Math.min(w - 1, x + 1)) << 2] * 0.299 + rgba[(row + Math.min(w - 1, x + 1)) << 2 + 1] * 0.587 + rgba[(row + Math.min(w - 1, x + 1)) << 2 + 2] * 0.114; const lumT = rgba[(Math.max(0, y - 1) * w + x) << 2] * 0.299 + rgba[(Math.max(0, y - 1) * w + x) << 2 + 1] * 0.587 + rgba[(Math.max(0, y - 1) * w + x) << 2 + 2] * 0.114; @@ -771,9 +849,9 @@ export function fastLocalInpaint( const p = idx << 2; const noise = randNorm() * grainStd; - const r = Math.round(baseR[idx] + hpR[idx] * 0.85 + noise); - const g = Math.round(baseG[idx] + hpG[idx] * 0.85 + noise); - const b = Math.round(baseB[idx] + hpB[idx] * 0.85 + noise); + const r = Math.round(baseR[idx] + hpR[idx] + noise); + const g = Math.round(baseG[idx] + hpG[idx] + noise); + const b = Math.round(baseB[idx] + hpB[idx] + noise); out[p] = r < 0 ? 0 : r > 255 ? 255 : r; out[p + 1] = g < 0 ? 0 : g > 255 ? 255 : g; @@ -818,14 +896,14 @@ export function cropPlate( const plane = w * h; const src = new Uint8Array(3 * plane); - const mask = new Uint8Array(plane).fill(255); + const mask = new Uint8Array(plane); // 0 = clean background to preserve, 255 = hole to inpaint for (let d = 0; d < plane; d++) { const s = d * 4; src[d] = small[s]; src[plane + d] = small[s + 1]; src[2 * plane + d] = small[s + 2]; if (smallHole[d]) { - mask[d] = 0; + mask[d] = 255; } } return { src, hole: mask }; @@ -864,6 +942,13 @@ export async function inpaintCrop( height: number, hole: Uint8Array ): Promise { + const maxSide = isMobile() ? 256 : INPAINT_MAX_SIDE; + let w = Math.max(8, Math.round(width)); + let h = Math.max(8, Math.round(height)); + const scale = Math.min(1, maxSide / Math.max(w, h)); + w = Math.max(8, Math.floor((w * scale) / 8) * 8); + h = Math.max(8, Math.floor((h * scale) / 8) * 8); + const patchKey = (hole as { patchKey?: string }).patchKey; let key: string; if (patchKey) { @@ -878,27 +963,95 @@ export async function inpaintCrop( holeHash = (holeHash * 31 + i) | 0; } } - key = `${width}x${height}_${holeCount}_${holeHash}`; + key = `${w}x${h}_${holeCount}_${holeHash}`; } if (patchCache.has(key)) { return patchCache.get(key)!.slice(); } - const patch = fastLocalInpaint(rgba, width, height, hole); - if (patchCache.size > 50) { - const first = patchCache.keys().next().value; - if (first) patchCache.delete(first); + // On mobile devices (iOS Safari/Chrome WebKit), running the 28MB ONNX neural model + // in WebAssembly exceeds the tab's strict Jetsam memory quota and triggers process termination (tab reload). + // Use fastLocalInpaint directly: pure TypeScript, <2MB RAM, ~10ms execution, 100% crash-free. + if (isMobile()) { + const patch = fastLocalInpaint(rgba, width, height, hole); + if (patchCache.size > 50) { + const first = patchCache.keys().next().value; + if (first) patchCache.delete(first); + } + patchCache.set(key, patch.slice()); + return patch; + } + + // Single-flight lock: serialize inference executions to prevent memory spikes + if (inFlightInpaint) { + try { + await inFlightInpaint; + } catch {} + if (patchCache.has(key)) { + return patchCache.get(key)!.slice(); + } + } + + const runPromise = (async (): Promise => { + const { src, hole: mask } = cropPlate(rgba, width, height, hole, w, h); + + let in0: Ort.Tensor | null = null; + let in1: Ort.Tensor | null = null; + let out: Record | null = null; + try { + const { ort, session } = await load(); + in0 = new ort.Tensor('uint8', src, [1, 3, h, w]); + in1 = new ort.Tensor('uint8', mask, [1, 1, h, w]); + out = await session.run({ + [session.inputNames[0]]: in0, + [session.inputNames[1]]: in1, + }); + const tensor = out[session.outputNames[0]]; + const oplane = w * h; + const data = tensor.data as Uint8Array; + const patch = new Uint8Array(4 * w * h); + for (let i = 0; i < oplane; i++) { + patch[i * 4] = data[i]; + patch[i * 4 + 1] = data[oplane + i]; + patch[i * 4 + 2] = data[2 * oplane + i]; + patch[i * 4 + 3] = 255; + } + if (patchCache.size > 50) { + const first = patchCache.keys().next().value; + if (first) patchCache.delete(first); + } + patchCache.set(key, patch.slice()); + return patch; + } catch (err) { + console.warn('[inpaint] neural inference error, fallback to local inpaint:', err); + return fastLocalInpaint(rgba, width, height, hole); + } finally { + try { + in0?.dispose?.(); + in1?.dispose?.(); + if (out) { + for (const k in out) out[k]?.dispose?.(); + } + } catch {} + } + })(); + + inFlightInpaint = runPromise; + try { + return await runPromise; + } finally { + if (inFlightInpaint === runPromise) { + inFlightInpaint = null; + } } - patchCache.set(key, patch.slice()); - return patch; } export function inpaintCropSize(width: number, height: number): { w: number; h: number } { - const maxSide = INPAINT_MAX_SIDE; + const maxSide = isMobile() ? 256 : INPAINT_MAX_SIDE; const scale = Math.min(1, maxSide / Math.max(1, Math.max(width, height))); return { - w: Math.max(1, Math.round(width * scale)), - h: Math.max(1, Math.round(height * scale)), + w: Math.max(8, Math.floor((Math.max(1, Math.round(width)) * scale) / 8) * 8), + h: Math.max(8, Math.floor((Math.max(1, Math.round(height)) * scale) / 8) * 8), }; }