3dtours/RecipesCam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(heal): expand crop margin, multi-directional donor search, smooth Dirichlet fallback, pre-warm LaMa

Browse Source
This commit is contained in:
3dtours
2026-10-16 22:18:19 +07:00
parent 285f402b72
commit 7f0344def4
3 changed files with 187 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/backend/src/server.ts
+4 -1
View File
@@ -1,4 +1,4 @@
import { runServerInpaint } from './inpaintService';
import { runServerInpaint, getInpaintSession } from './inpaintService';
import { recipeFile } from './recipeFile';
import { sendVerificationMail } from './mailer';
import { placeName } from './place';
@@ -426,6 +426,9 @@ async function lookupGeo(ip: string): Promise<GeoPlace | null> {
// ---- routes ----
app.get('/api/health', async () => ({ ok: true }));
// Pre-warm ONNX inpainting model on startup
void getInpaintSession();
// AI Inpainting endpoint (Hybrid Option 2)
app.post('/api/inpaint', { bodyLimit: 10 * 1024 * 1024 }, async (req, reply) => {
const b = bodyOf(req) as {
docker/frontend/src/engine/exportEngine.ts
+1 -1
View File
@@ -1079,7 +1079,7 @@ async function applyHealToBase(
const cSpanW = mx1 - mx0;
const cSpanH = my1 - my0;
const cSpan = Math.max(cSpanW, cSpanH);
const margin = Math.max(24, Math.min(96, Math.round(cWidest * 2 + 16)));
const margin = Math.max(48, Math.min(192, Math.round(Math.max(cWidest * 2.5, cSpan * 0.4) + 24)));
const srcLeft = Math.max(boxLeft, Math.floor(mx0 - margin));
const srcTop = Math.max(boxTop, Math.floor(my0 - margin));
const srcRight = Math.min(boxRight, Math.ceil(mx1 + margin));
docker/frontend/src/engine/inpaint.ts
+182 -134
View File
@@ -207,14 +207,16 @@ export function applyCoherentTexture(
}
/**
* FAST ENGINE: Clean Donor Patch Search + Seamless Poisson Blending (SOR Laplace Solver).
* FAST ENGINE: Multi-Directional Clean Donor Search + Seamless Poisson Blending (SOR Laplace Solver).
* Industry benchmark for spot healing (Adobe Photoshop / Lightroom standard):
* 1. Evaluates candidate clean photographic donor patches surrounding the hole.
* 1. Evaluates candidate clean photographic donor patches surrounding the hole across 32 directions.
* 2. Selects the donor patch with optimal boundary color & gradient alignment and 0% hole overlap.
* 3. Solves the Poisson Laplace equation ∇² E = 0 for the error/difference field E = I_clean - I_donor
* using Successive Over-Relaxation (SOR, omega = 1.75).
* 4. Yields 100% genuine photographic texture (waves, ripples, grain, wood grain, fabric)
* seamlessly blended with zero edge artifacts, zero comb lines, and zero synthetic blur.
* 5. Robust Fallback: If no single translation offset fits (e.g. huge hole), smoothly relaxes via
* Dirichlet harmonic field ∇² I = 0 with zero Voronoi facets or hard stripes.
* Runs in pure TypeScript in ~5-15ms with <2MB memory.
*/
export function fastLocalInpaint(
@@ -250,10 +252,10 @@ export function fastLocalInpaint(
return res;
}
// 1. Hole dimensions and search radius
// 1. Hole dimensions and search parameters
const holeW = xmax - xmin + 1;
const holeH = ymax - ymin + 1;
const rad = Math.max(holeW, holeH) * 0.5;
const minStroke = Math.min(holeW, holeH);
// 2. Identify clean boundary band and immediate hole boundary pixels
const band: number[] = [];
@@ -295,18 +297,16 @@ export function fastLocalInpaint(
}
}
// 3. Clean Donor Patch Search
// 3. Multi-Directional Clean Donor Search
let bestDx = 0, bestDy = 0;
let minCost = Infinity;
let foundDonor = false;
const minD = Math.max(4, Math.round(rad * 1.2));
const maxD = Math.max(minD + 10, Math.min(Math.round(rad * 3.5), Math.max(w, h)));
const dSteps = [minD, Math.round(minD * 1.5), Math.round(minD * 2.0), Math.round(minD * 2.6), maxD];
const numAngles = 24;
const minD = Math.max(6, Math.round(minStroke * 0.6));
const maxD = Math.min(Math.round(Math.max(holeW, holeH) * 2.5), Math.max(w, h));
const numAngles = 32;
for (const d of dSteps) {
if (d > maxD) continue;
for (let d = minD; d <= maxD; d += 4) {
for (let a = 0; a < numAngles; a++) {
const angle = (a * 2 * Math.PI) / numAngles;
const dx = Math.round(Math.cos(angle) * d);
@@ -316,7 +316,12 @@ export function fastLocalInpaint(
let valid = true;
for (let y = ymin; y <= ymax && valid; y++) {
const sy = y + dy;
if (sy < 0 || sy >= h) { valid = false; break; }
if (sy < 0 || sy >= h) {
for (let x = xmin; x <= xmax; x++) {
if (hole[y * w + x]) { valid = false; break; }
}
continue;
}
const row = y * w;
const sRow = sy * w;
for (let x = xmin; x <= xmax; x++) {
@@ -367,12 +372,15 @@ export function fastLocalInpaint(
}
}
// 4. Fill donor colors
const donorR = new Uint8Array(total);
const donorG = new Uint8Array(total);
const donorB = new Uint8Array(total);
const out = new Uint8Array(rgba.length);
out.set(rgba);
if (foundDonor) {
// 4A. SEAMLESS POISSON BLENDING WITH CLEAN PHOTOGRAPHIC DONOR
const donorR = new Uint8Array(total);
const donorG = new Uint8Array(total);
const donorB = new Uint8Array(total);
for (let y = ymin; y <= ymax; y++) {
const row = y * w;
const sRow = (y + bestDy) * w;
@@ -386,151 +394,191 @@ export function fastLocalInpaint(
donorB[idx] = rgba[sp + 2];
}
}
} else {
// Robust fallback: sample nearest clean boundary pixel
const fallbackBand = band.length > 0 ? band : [0];
const diffR = new Float32Array(total);
const diffG = new Float32Array(total);
const diffB = new Float32Array(total);
let avgEdgeDiffR = 0, avgEdgeDiffG = 0, avgEdgeDiffB = 0;
let edgeCount = 0;
for (let i = 0; i < holeBoundary.length; i++) {
const idx = holeBoundary[i];
const x = idx % w;
const y = (idx / w) | 0;
let cR = 0, cG = 0, cB = 0, cN = 0;
if (x > 0 && clean[idx - 1]) { const p = (idx - 1) << 2; cR += rgba[p]; cG += rgba[p + 1]; cB += rgba[p + 2]; cN++; }
if (x < w - 1 && clean[idx + 1]) { const p = (idx + 1) << 2; cR += rgba[p]; cG += rgba[p + 1]; cB += rgba[p + 2]; cN++; }
if (y > 0 && clean[idx - w]) { const p = (idx - w) << 2; cR += rgba[p]; cG += rgba[p + 1]; cB += rgba[p + 2]; cN++; }
if (y < h - 1 && clean[idx + w]) { const p = (idx + w) << 2; cR += rgba[p]; cG += rgba[p + 1]; cB += rgba[p + 2]; cN++; }
if (cN > 0) {
const er = (cR / cN) - donorR[idx];
const eg = (cG / cN) - donorG[idx];
const eb = (cB / cN) - donorB[idx];
diffR[idx] = er; diffG[idx] = eg; diffB[idx] = eb;
avgEdgeDiffR += er; avgEdgeDiffG += eg; avgEdgeDiffB += eb;
edgeCount++;
}
}
if (edgeCount > 0) {
avgEdgeDiffR /= edgeCount;
avgEdgeDiffG /= edgeCount;
avgEdgeDiffB /= edgeCount;
}
for (let y = ymin; y <= ymax; y++) {
const row = y * w;
for (let x = xmin; x <= xmax; x++) {
const idx = row + x;
if (!hole[idx]) continue;
let cIdx = fallbackBand[0];
let cDist = Infinity;
for (let i = 0; i < fallbackBand.length; i++) {
const bx = fallbackBand[i] % w;
const by = (fallbackBand[i] / w) | 0;
const dist = (x - bx) * (x - bx) + (y - by) * (y - by);
if (dist < cDist) { cDist = dist; cIdx = fallbackBand[i]; }
if (hole[idx] && diffR[idx] === 0 && diffG[idx] === 0 && diffB[idx] === 0) {
diffR[idx] = avgEdgeDiffR;
diffG[idx] = avgEdgeDiffG;
diffB[idx] = avgEdgeDiffB;
}
const cp = cIdx << 2;
donorR[idx] = rgba[cp];
donorG[idx] = rgba[cp + 1];
donorB[idx] = rgba[cp + 2];
}
}
}
// 5. Difference Field Setup for Poisson Seamless Blending
const diffR = new Float32Array(total);
const diffG = new Float32Array(total);
const diffB = new Float32Array(total);
const omega = 1.75;
const oneMinusOmega = 1.0 - omega;
const iterations = 28;
let avgEdgeDiffR = 0, avgEdgeDiffG = 0, avgEdgeDiffB = 0;
let edgeCount = 0;
for (let it = 0; it < iterations; it++) {
for (let y = ymin; y <= ymax; y++) {
const row = y * w;
for (let x = xmin; x <= xmax; x++) {
const idx = row + x;
if (!hole[idx]) continue;
for (let i = 0; i < holeBoundary.length; i++) {
const idx = holeBoundary[i];
const x = idx % w;
const y = (idx / w) | 0;
const rR = x < w - 1 ? diffR[idx + 1] : (x > 0 ? diffR[idx - 1] : diffR[idx]);
const rL = x > 0 ? diffR[idx - 1] : (x < w - 1 ? diffR[idx + 1] : diffR[idx]);
const rD = y < h - 1 ? diffR[idx + w] : (y > 0 ? diffR[idx - w] : diffR[idx]);
const rU = y > 0 ? diffR[idx - w] : (y < h - 1 ? diffR[idx + w] : diffR[idx]);
let cR = 0, cG = 0, cB = 0, cN = 0;
if (x > 0 && clean[idx - 1]) {
const p = (idx - 1) << 2;
cR += rgba[p]; cG += rgba[p + 1]; cB += rgba[p + 2]; cN++;
}
if (x < w - 1 && clean[idx + 1]) {
const p = (idx + 1) << 2;
cR += rgba[p]; cG += rgba[p + 1]; cB += rgba[p + 2]; cN++;
}
if (y > 0 && clean[idx - w]) {
const p = (idx - w) << 2;
cR += rgba[p]; cG += rgba[p + 1]; cB += rgba[p + 2]; cN++;
}
if (y < h - 1 && clean[idx + w]) {
const p = (idx + w) << 2;
cR += rgba[p]; cG += rgba[p + 1]; cB += rgba[p + 2]; cN++;
}
diffR[idx] = oneMinusOmega * diffR[idx] + omega * (rR + rL + rD + rU) * 0.25;
if (cN > 0) {
const er = (cR / cN) - donorR[idx];
const eg = (cG / cN) - donorG[idx];
const eb = (cB / cN) - donorB[idx];
diffR[idx] = er;
diffG[idx] = eg;
diffB[idx] = eb;
avgEdgeDiffR += er;
avgEdgeDiffG += eg;
avgEdgeDiffB += eb;
edgeCount++;
}
}
const gR = x < w - 1 ? diffG[idx + 1] : (x > 0 ? diffG[idx - 1] : diffG[idx]);
const gL = x > 0 ? diffG[idx - 1] : (x < w - 1 ? diffG[idx + 1] : diffG[idx]);
const gD = y < h - 1 ? diffG[idx + w] : (y > 0 ? diffG[idx - w] : diffG[idx]);
const gU = y > 0 ? diffG[idx - w] : (y < h - 1 ? diffG[idx + w] : diffG[idx]);
if (edgeCount > 0) {
avgEdgeDiffR /= edgeCount;
avgEdgeDiffG /= edgeCount;
avgEdgeDiffB /= edgeCount;
}
diffG[idx] = oneMinusOmega * diffG[idx] + omega * (gR + gL + gD + gU) * 0.25;
for (let y = ymin; y <= ymax; y++) {
const row = y * w;
for (let x = xmin; x <= xmax; x++) {
const idx = row + x;
if (hole[idx] && diffR[idx] === 0 && diffG[idx] === 0 && diffB[idx] === 0) {
diffR[idx] = avgEdgeDiffR;
diffG[idx] = avgEdgeDiffG;
diffB[idx] = avgEdgeDiffB;
const bR = x < w - 1 ? diffB[idx + 1] : (x > 0 ? diffB[idx - 1] : diffB[idx]);
const bL = x > 0 ? diffB[idx - 1] : (x < w - 1 ? diffB[idx + 1] : diffB[idx]);
const bD = y < h - 1 ? diffB[idx + w] : (y > 0 ? diffB[idx - w] : diffB[idx]);
const bU = y > 0 ? diffB[idx - w] : (y < h - 1 ? diffB[idx + w] : diffB[idx]);
diffB[idx] = oneMinusOmega * diffB[idx] + omega * (bR + bL + bD + bU) * 0.25;
}
}
}
}
// 6. Successive Over-Relaxation (SOR) solver for Laplace equation ∇² E = 0
const omega = 1.75;
const oneMinusOmega = 1.0 - omega;
const iterations = 28;
for (let it = 0; it < iterations; it++) {
for (let y = ymin; y <= ymax; y++) {
const row = y * w;
for (let x = xmin; x <= xmax; x++) {
const idx = row + x;
if (!hole[idx]) continue;
const rR = x < w - 1 ? diffR[idx + 1] : (x > 0 ? diffR[idx - 1] : diffR[idx]);
const rL = x > 0 ? diffR[idx - 1] : (x < w - 1 ? diffR[idx + 1] : diffR[idx]);
const rD = y < h - 1 ? diffR[idx + w] : (y > 0 ? diffR[idx - w] : diffR[idx]);
const rU = y > 0 ? diffR[idx - w] : (y < h - 1 ? diffR[idx + w] : diffR[idx]);
const avgR = (rR + rL + rD + rU) * 0.25;
diffR[idx] = oneMinusOmega * diffR[idx] + omega * avgR;
const gR = x < w - 1 ? diffG[idx + 1] : (x > 0 ? diffG[idx - 1] : diffG[idx]);
const gL = x > 0 ? diffG[idx - 1] : (x < w - 1 ? diffG[idx + 1] : diffG[idx]);
const gD = y < h - 1 ? diffG[idx + w] : (y > 0 ? diffG[idx - w] : diffG[idx]);
const gU = y > 0 ? diffG[idx - w] : (y < h - 1 ? diffG[idx + w] : diffG[idx]);
const avgG = (gR + gL + gD + gU) * 0.25;
diffG[idx] = oneMinusOmega * diffG[idx] + omega * avgG;
const bR = x < w - 1 ? diffB[idx + 1] : (x > 0 ? diffB[idx - 1] : diffB[idx]);
const bL = x > 0 ? diffB[idx - 1] : (x < w - 1 ? diffB[idx + 1] : diffB[idx]);
const bD = y < h - 1 ? diffB[idx + w] : (y > 0 ? diffB[idx - w] : diffB[idx]);
const bU = y > 0 ? diffB[idx - w] : (y < h - 1 ? diffB[idx + w] : diffB[idx]);
const avgB = (bR + bL + bD + bU) * 0.25;
diffB[idx] = oneMinusOmega * diffB[idx] + omega * avgB;
const p = idx << 2;
const r = Math.round(donorR[idx] + diffR[idx]);
const g = Math.round(donorG[idx] + diffG[idx]);
const b = Math.round(donorB[idx] + diffB[idx]);
out[p] = r < 0 ? 0 : r > 255 ? 255 : r;
out[p + 1] = g < 0 ? 0 : g > 255 ? 255 : g;
out[p + 2] = b < 0 ? 0 : b > 255 ? 255 : b;
out[p + 3] = 255;
}
}
}
} else {
// 4B. ROBUST DIRICHLET HARMONIC RELAXATION FALLBACK (smooth, continuous, zero Voronoi facets)
const baseR = new Float32Array(total);
const baseG = new Float32Array(total);
const baseB = new Float32Array(total);
// 7. Assemble final healed plate
const out = new Uint8Array(rgba.length);
out.set(rgba);
for (let i = 0; i < total; i++) {
if (clean[i]) {
const p = i << 2;
baseR[i] = rgba[p];
baseG[i] = rgba[p + 1];
baseB[i] = rgba[p + 2];
}
}
for (let y = ymin; y <= ymax; y++) {
const row = y * w;
for (let x = xmin; x <= xmax; x++) {
const idx = row + x;
if (!hole[idx]) continue;
for (let y = ymin; y <= ymax; y++) {
const row = y * w;
for (let x = xmin; x <= xmax; x++) {
const idx = row + x;
if (clean[idx]) continue;
const p = idx << 2;
const r = Math.round(donorR[idx] + diffR[idx]);
const g = Math.round(donorG[idx] + diffG[idx]);
const b = Math.round(donorB[idx] + diffB[idx]);
let xl = x, xr = x, yt = y, yb = y;
while (xl >= 0 && !clean[row + xl]) xl--;
while (xr < w && !clean[row + xr]) xr++;
while (yt >= 0 && !clean[yt * w + x]) yt--;
while (yb < h && !clean[yb * w + x]) yb++;
out[p] = r < 0 ? 0 : r > 255 ? 255 : r;
out[p + 1] = g < 0 ? 0 : g > 255 ? 255 : g;
out[p + 2] = b < 0 ? 0 : b > 255 ? 255 : b;
out[p + 3] = 255;
let sumR = 0, sumG = 0, sumB = 0, sumW = 0;
if (xl >= 0) { const d = x - xl; const wt = 1.0 / (d * d); const p = (row + xl) << 2; sumR += rgba[p] * wt; sumG += rgba[p + 1] * wt; sumB += rgba[p + 2] * wt; sumW += wt; }
if (xr < w) { const d = xr - x; const wt = 1.0 / (d * d); const p = (row + xr) << 2; sumR += rgba[p] * wt; sumG += rgba[p + 1] * wt; sumB += rgba[p + 2] * wt; sumW += wt; }
if (yt >= 0) { const d = y - yt; const wt = 1.0 / (d * d); const p = (yt * w + x) << 2; sumR += rgba[p] * wt; sumG += rgba[p + 1] * wt; sumB += rgba[p + 2] * wt; sumW += wt; }
if (yb < h) { const d = yb - y; const wt = 1.0 / (d * d); const p = (yb * w + x) << 2; sumR += rgba[p] * wt; sumG += rgba[p + 1] * wt; sumB += rgba[p + 2] * wt; sumW += wt; }
if (sumW > 0) {
baseR[idx] = sumR / sumW;
baseG[idx] = sumG / sumW;
baseB[idx] = sumB / sumW;
}
}
}
const omega = 1.75;
const oneMinusOmega = 1.0 - omega;
for (let it = 0; it < 32; it++) {
for (let y = ymin; y <= ymax; y++) {
const row = y * w;
for (let x = xmin; x <= xmax; x++) {
const idx = row + x;
if (clean[idx]) continue;
const rR = x < w - 1 ? baseR[idx + 1] : (x > 0 ? baseR[idx - 1] : baseR[idx]);
const rL = x > 0 ? baseR[idx - 1] : (x < w - 1 ? baseR[idx + 1] : baseR[idx]);
const rD = y < h - 1 ? baseR[idx + w] : (y > 0 ? baseR[idx - w] : baseR[idx]);
const rU = y > 0 ? baseR[idx - w] : (y < h - 1 ? baseR[idx + w] : baseR[idx]);
baseR[idx] = oneMinusOmega * baseR[idx] + omega * (rR + rL + rD + rU) * 0.25;
const gR = x < w - 1 ? baseG[idx + 1] : (x > 0 ? baseG[idx - 1] : baseG[idx]);
const gL = x > 0 ? baseG[idx - 1] : (x < w - 1 ? baseG[idx + 1] : baseG[idx]);
const gD = y < h - 1 ? baseG[idx + w] : (y > 0 ? baseG[idx - w] : baseG[idx]);
const gU = y > 0 ? baseG[idx - w] : (y < h - 1 ? baseG[idx + w] : baseG[idx]);
baseG[idx] = oneMinusOmega * baseG[idx] + omega * (gR + gL + gD + gU) * 0.25;
const bR = x < w - 1 ? baseB[idx + 1] : (x > 0 ? baseB[idx - 1] : baseB[idx]);
const bL = x > 0 ? baseB[idx - 1] : (x < w - 1 ? baseB[idx + 1] : baseB[idx]);
const bD = y < h - 1 ? baseB[idx + w] : (y > 0 ? baseB[idx - w] : baseB[idx]);
const bU = y > 0 ? baseB[idx - w] : (y < h - 1 ? baseB[idx + w] : baseB[idx]);
baseB[idx] = oneMinusOmega * baseB[idx] + omega * (bR + bL + bD + bU) * 0.25;
}
}
}
for (let y = ymin; y <= ymax; y++) {
const row = y * w;
for (let x = xmin; x <= xmax; x++) {
const idx = row + x;
if (!hole[idx]) continue;
const p = idx << 2;
const r = Math.round(baseR[idx]);
const g = Math.round(baseG[idx]);
const b = Math.round(baseB[idx]);
out[p] = r < 0 ? 0 : r > 255 ? 255 : r;
out[p + 1] = g < 0 ? 0 : g > 255 ? 255 : g;
out[p + 2] = b < 0 ? 0 : b > 255 ? 255 : b;
out[p + 3] = 255;
}
}
}
@@ -693,7 +741,7 @@ export async function inpaintCrop(
const maskB64 = uint8ToBase64(mask);
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 8500);
const timeoutId = setTimeout(() => controller.abort(), 15000);
const res = await fetch('/api/inpaint', {
method: 'POST',