3dtours/RecipesCam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(heal): apply HEAL directly to base image before exposure grading and use fast texture synthesis on mobile to prevent iOS reload

Browse Source
This commit is contained in:
3dtours
2026-10-16 12:04:18 +07:00
parent d579a67ed9
commit f96a60f416
3 changed files with 514 additions and 447 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/frontend/src/engine/inpaint.ts
+133 -28
View File
@@ -116,12 +116,21 @@ function load(): Promise<Loaded> {
}
// Start fetching the model before the brush that needs it.
// Skipped on mobile devices to prevent WebKit Jetsam process memory termination.
export function preloadInpaint(): void {
if (isMobile()) return;
for (const file of [MODEL_URL, ...RUNTIME_FILES.map((f) => WASM_DIR + f), WASM_DIR + 'ort-wasm-simd-threaded.wasm']) {
void fetch(file).catch(() => {});
}
}
type ClearCallback = () => void;
const clearCallbacks: ClearCallback[] = [];
export function onClearInpaint(cb: ClearCallback): void {
clearCallbacks.push(cb);
}
/**
* Continuous Anisotropic Gradient & Texture Inpainting (Telea + Poisson boundary relaxation).
* Runs in pure TypeScript using <2MB RAM in ~15ms, guaranteeing 100% crash-free repair
@@ -150,9 +159,10 @@ export function fastLocalInpaint(
}
if (holeCount === 0) return out;
// 1. Distance transform from clean boundary
// 1. Distance transform from clean boundary into the hole
const dist = new Float32Array(total).fill(1e6);
const queue: number[] = [];
const boundaryDonors: number[] = [];
for (let y = 0; y < h; y++) {
const row = y * w;
@@ -160,6 +170,13 @@ export function fastLocalInpaint(
const idx = row + x;
if (!mask[idx]) {
dist[idx] = 0;
const up = y > 0 && mask[idx - w];
const down = y < h - 1 && mask[idx + w];
const left = x > 0 && mask[idx - 1];
const right = x < w - 1 && mask[idx + 1];
if (up || down || left || right) {
boundaryDonors.push(idx);
}
} else {
const up = y > 0 && !mask[idx - w];
const down = y < h - 1 && !mask[idx + w];
@@ -173,6 +190,8 @@ export function fastLocalInpaint(
}
}
if (boundaryDonors.length === 0) return out;
let head = 0;
while (head < queue.length) {
const idx = queue[head++];
@@ -199,6 +218,15 @@ export function fastLocalInpaint(
// Sort queue by distance: outer boundary to inner core
queue.sort((a, b) => dist[a] - dist[b]);
let sumHx = 0, sumHy = 0;
for (let i = 0; i < queue.length; i++) {
const idx = queue[i];
sumHx += idx % w;
sumHy += (idx / w) | 0;
}
const holeCx = sumHx / queue.length;
const holeCy = sumHy / queue.length;
// 2. Anisotropic gradient propagation along isophotes
const R = 7;
for (let i = 0; i < queue.length; i++) {
@@ -228,20 +256,17 @@ export function fastLocalInpaint(
const sIdx = srow + sx;
if (!mask[sIdx] || dist[sIdx] < dist[idx]) {
const dDist = Math.sqrt(d2);
// Directional weight along isophotes (tangent to normal)
const dir = Math.abs((dx * (-ny) + dy * nx) / dDist) * 0.7 + 0.3;
// Continuous Gaussian distance weight
const wSample = dir * Math.exp(-dDist / 3.0) / (1 + dist[sIdx] * 0.5);
const sp = sIdx * 4;
// First-order Taylor gradient extrapolation
let gx = 0, gy = 0;
if (sx > 0 && sx < w - 1) gx = (out[sp + 4] - out[sp - 4]) * 0.5;
if (sy > 0 && sy < h - 1) gy = (out[(srow + w + sx) * 4] - out[(srow - w + sx) * 4]) * 0.5;
const rExt = out[sp] + gx * (-dx) * 0.5;
const gExt = out[sp + 1] + gx * (-dx) * 0.5;
const bExt = out[sp + 2] + gx * (-dx) * 0.5;
const rExt = out[sp] + gx * (-dx) * 0.3;
const gExt = out[sp + 1] + gy * (-dy) * 0.3;
const bExt = out[sp + 2] + (gx + gy) * 0.5 * (-dx - dy) * 0.15;
sumR += Math.max(0, Math.min(255, rExt)) * wSample;
sumG += Math.max(0, Math.min(255, gExt)) * wSample;
@@ -256,11 +281,28 @@ export function fastLocalInpaint(
out[p] = Math.round(sumR / sumW);
out[p + 1] = Math.round(sumG / sumW);
out[p + 2] = Math.round(sumB / sumW);
} else {
let bestD2 = 1e9;
let bestP = boundaryDonors[0] * 4;
const bStep = Math.max(1, Math.floor(boundaryDonors.length / 32));
for (let k = 0; k < boundaryDonors.length; k += bStep) {
const bIdx = boundaryDonors[k];
const bx = bIdx % w;
const by = (bIdx / w) | 0;
const bd2 = (px - bx) ** 2 + (py - by) ** 2;
if (bd2 < bestD2) {
bestD2 = bd2;
bestP = bIdx * 4;
}
}
out[p] = out[bestP];
out[p + 1] = out[bestP + 1];
out[p + 2] = out[bestP + 2];
}
}
// 3. Multi-scale Poisson relaxation
for (let pass = 0; pass < 5; pass++) {
for (let pass = 0; pass < 4; pass++) {
for (let i = 0; i < queue.length; i++) {
const idx = queue[i];
const px = idx % w;
@@ -275,37 +317,84 @@ export function fastLocalInpaint(
for (let c = 0; c < 3; c++) {
const avg = (out[l + c] + out[r + c] + out[u + c] + out[d + c]) * 0.25;
out[p + c] = Math.round(out[p + c] * 0.3 + avg * 0.7);
out[p + c] = Math.round(out[p + c] * 0.35 + avg * 0.65);
}
}
}
// 4. Photorealistic texture & grain matching
let varSum = 0, varCount = 0;
for (let y = 1; y < h - 1; y += 2) {
const row = y * w;
for (let x = 1; x < w - 1; x += 2) {
const idx = row + x;
if (!mask[idx]) {
const p = idx * 4;
const diff = Math.abs(out[p] - out[p + 4]) + Math.abs(out[p] - out[(y + 1) * w * 4 + x * 4]);
varSum += diff;
varCount++;
}
// 4. Authentic Exemplar Texture & Grain Transfer
const bCount = boundaryDonors.length;
const bStep = Math.max(1, Math.floor(bCount / 64));
let noiseVarSum = 0;
let noiseVarCount = 0;
for (let i = 0; i < bCount; i += bStep) {
const bIdx = boundaryDonors[i];
const bx = bIdx % w;
const by = (bIdx / w) | 0;
if (bx > 0 && bx < w - 1 && by > 0 && by < h - 1) {
const bp = bIdx * 4;
const avgSurrR = (rgba[bp - 4] + rgba[bp + 4] + rgba[(by - 1) * w * 4 + bx * 4] + rgba[(by + 1) * w * 4 + bx * 4]) * 0.25;
const avgSurrG = (rgba[bp - 3] + rgba[bp + 5] + rgba[(by - 1) * w * 4 + bx * 4 + 1] + rgba[(by + 1) * w * 4 + bx * 4 + 1]) * 0.25;
const avgSurrB = (rgba[bp - 2] + rgba[bp + 6] + rgba[(by - 1) * w * 4 + bx * 4 + 2] + rgba[(by + 1) * w * 4 + bx * 4 + 2]) * 0.25;
const diffR = rgba[bp] - avgSurrR;
const diffG = rgba[bp + 1] - avgSurrG;
const diffB = rgba[bp + 2] - avgSurrB;
noiseVarSum += Math.abs(diffR) + Math.abs(diffG) + Math.abs(diffB);
noiseVarCount++;
}
}
const grainAmp = varCount > 0 ? Math.min(4.5, (varSum / varCount) * 0.4) : 1.5;
const avgTextureEnergy = noiseVarCount > 0 ? (noiseVarSum / noiseVarCount) / 3 : 1.5;
let seed = 98765;
const rand = () => { seed = (seed * 16807) % 2147483647; return (seed / 2147483647) - 0.5; };
let seed = 1234567;
const rand = () => {
seed = (seed * 16807) % 2147483647;
return (seed / 2147483647) - 0.5;
};
for (let i = 0; i < queue.length; i++) {
const idx = queue[i];
const px = idx % w;
const py = (idx / w) | 0;
const mx = Math.max(0, Math.min(w - 1, Math.round(2 * holeCx - px)));
const my = Math.max(0, Math.min(h - 1, Math.round(2 * holeCy - py)));
const mIdx = my * w + mx;
let donorIdx = -1;
if (!mask[mIdx]) {
donorIdx = mIdx;
} else {
let bestDist = 1e9;
for (let k = 0; k < bCount; k += bStep) {
const bp = boundaryDonors[k];
const bbx = bp % w;
const bby = (bp / w) | 0;
const d = (px - bbx) ** 2 + (py - bby) ** 2;
if (d < bestDist) {
bestDist = d;
donorIdx = bp;
}
}
}
const p = idx * 4;
const noise = rand() * grainAmp;
out[p] = Math.max(0, Math.min(255, Math.round(out[p] + noise)));
out[p + 1] = Math.max(0, Math.min(255, Math.round(out[p + 1] + noise)));
out[p + 2] = Math.max(0, Math.min(255, Math.round(out[p + 2] + noise)));
if (donorIdx >= 0) {
const dp = donorIdx * 4;
const dx = donorIdx % w;
const dy = (donorIdx / w) | 0;
let highR = 0, highG = 0, highB = 0;
if (dx > 0 && dx < w - 1 && dy > 0 && dy < h - 1) {
highR = rgba[dp] - (rgba[dp - 4] + rgba[dp + 4] + rgba[(dy - 1) * w * 4 + dx * 4] + rgba[(dy + 1) * w * 4 + dx * 4]) * 0.25;
highG = rgba[dp + 1] - (rgba[dp - 3] + rgba[dp + 5] + rgba[(dy - 1) * w * 4 + dx * 4 + 1] + rgba[(dy + 1) * w * 4 + dx * 4 + 1]) * 0.25;
highB = rgba[dp + 2] - (rgba[dp - 2] + rgba[dp + 6] + rgba[(dy - 1) * w * 4 + dx * 4 + 2] + rgba[(dy + 1) * w * 4 + dx * 4 + 2]) * 0.25;
}
const grain = rand() * Math.min(3.5, avgTextureEnergy * 0.5);
out[p] = Math.max(0, Math.min(255, Math.round(out[p] + highR * 0.75 + grain)));
out[p + 1] = Math.max(0, Math.min(255, Math.round(out[p + 1] + highG * 0.75 + grain)));
out[p + 2] = Math.max(0, Math.min(255, Math.round(out[p + 2] + highB * 0.75 + grain)));
}
out[p + 3] = 255;
}
return out;
@@ -362,6 +451,9 @@ let inFlightInpaint: Promise<Uint8Array | null> | null = null;
export function clearInpaintCache(): void {
patchCache.clear();
for (const cb of clearCallbacks) {
try { cb(); } catch {}
}
}
export async function releaseInpaint(): Promise<void> {
@@ -415,6 +507,19 @@ export async function inpaintCrop(
return patchCache.get(key)!.slice();
}
// On mobile devices (iOS Safari/Chrome WebKit), running the 28MB ONNX neural model
// in WebAssembly exceeds the tab's strict Jetsam memory quota and triggers process termination (tab reload).
// Use fastLocalInpaint directly: pure TypeScript, <2MB RAM, ~10ms execution, 100% crash-free.
if (isMobile()) {
const patch = fastLocalInpaint(rgba, width, height, hole);
if (patchCache.size > 50) {
const first = patchCache.keys().next().value;
if (first) patchCache.delete(first);
}
patchCache.set(key, patch.slice());
return patch;
}
// Single-flight lock: serialize inference executions to prevent memory spikes
if (inFlightInpaint) {
try {