diff --git a/docker/backend/src/server.ts b/docker/backend/src/server.ts index ee895af..5e80d72 100644 --- a/docker/backend/src/server.ts +++ b/docker/backend/src/server.ts @@ -1,4 +1,4 @@ -import { runServerInpaint } from './inpaintService'; +import { runServerInpaint, getInpaintSession } from './inpaintService'; import { recipeFile } from './recipeFile'; import { sendVerificationMail } from './mailer'; import { placeName } from './place'; @@ -426,6 +426,9 @@ async function lookupGeo(ip: string): Promise { // ---- routes ---- app.get('/api/health', async () => ({ ok: true })); +// Pre-warm ONNX inpainting model on startup +void getInpaintSession(); + // AI Inpainting endpoint (Hybrid Option 2) app.post('/api/inpaint', { bodyLimit: 10 * 1024 * 1024 }, async (req, reply) => { const b = bodyOf(req) as { diff --git a/docker/frontend/src/engine/exportEngine.ts b/docker/frontend/src/engine/exportEngine.ts index 19c953b..ecbc252 100644 --- a/docker/frontend/src/engine/exportEngine.ts +++ b/docker/frontend/src/engine/exportEngine.ts @@ -1079,7 +1079,7 @@ async function applyHealToBase( const cSpanW = mx1 - mx0; const cSpanH = my1 - my0; const cSpan = Math.max(cSpanW, cSpanH); - const margin = Math.max(24, Math.min(96, Math.round(cWidest * 2 + 16))); + const margin = Math.max(48, Math.min(192, Math.round(Math.max(cWidest * 2.5, cSpan * 0.4) + 24))); const srcLeft = Math.max(boxLeft, Math.floor(mx0 - margin)); const srcTop = Math.max(boxTop, Math.floor(my0 - margin)); const srcRight = Math.min(boxRight, Math.ceil(mx1 + margin)); diff --git a/docker/frontend/src/engine/inpaint.ts b/docker/frontend/src/engine/inpaint.ts index 1a92723..d9b0ec8 100644 --- a/docker/frontend/src/engine/inpaint.ts +++ b/docker/frontend/src/engine/inpaint.ts @@ -207,14 +207,16 @@ export function applyCoherentTexture( } /** - * FAST ENGINE: Clean Donor Patch Search + Seamless Poisson Blending (SOR Laplace Solver). + * FAST ENGINE: Multi-Directional Clean Donor Search + Seamless Poisson Blending (SOR Laplace Solver). * Industry benchmark for spot healing (Adobe Photoshop / Lightroom standard): - * 1. Evaluates candidate clean photographic donor patches surrounding the hole. + * 1. Evaluates candidate clean photographic donor patches surrounding the hole across 32 directions. * 2. Selects the donor patch with optimal boundary color & gradient alignment and 0% hole overlap. * 3. Solves the Poisson Laplace equation ∇² E = 0 for the error/difference field E = I_clean - I_donor * using Successive Over-Relaxation (SOR, omega = 1.75). * 4. Yields 100% genuine photographic texture (waves, ripples, grain, wood grain, fabric) * seamlessly blended with zero edge artifacts, zero comb lines, and zero synthetic blur. + * 5. Robust Fallback: If no single translation offset fits (e.g. huge hole), smoothly relaxes via + * Dirichlet harmonic field ∇² I = 0 with zero Voronoi facets or hard stripes. * Runs in pure TypeScript in ~5-15ms with <2MB memory. */ export function fastLocalInpaint( @@ -250,10 +252,10 @@ export function fastLocalInpaint( return res; } - // 1. Hole dimensions and search radius + // 1. Hole dimensions and search parameters const holeW = xmax - xmin + 1; const holeH = ymax - ymin + 1; - const rad = Math.max(holeW, holeH) * 0.5; + const minStroke = Math.min(holeW, holeH); // 2. Identify clean boundary band and immediate hole boundary pixels const band: number[] = []; @@ -295,18 +297,16 @@ export function fastLocalInpaint( } } - // 3. Clean Donor Patch Search + // 3. Multi-Directional Clean Donor Search let bestDx = 0, bestDy = 0; let minCost = Infinity; let foundDonor = false; - const minD = Math.max(4, Math.round(rad * 1.2)); - const maxD = Math.max(minD + 10, Math.min(Math.round(rad * 3.5), Math.max(w, h))); - const dSteps = [minD, Math.round(minD * 1.5), Math.round(minD * 2.0), Math.round(minD * 2.6), maxD]; - const numAngles = 24; + const minD = Math.max(6, Math.round(minStroke * 0.6)); + const maxD = Math.min(Math.round(Math.max(holeW, holeH) * 2.5), Math.max(w, h)); + const numAngles = 32; - for (const d of dSteps) { - if (d > maxD) continue; + for (let d = minD; d <= maxD; d += 4) { for (let a = 0; a < numAngles; a++) { const angle = (a * 2 * Math.PI) / numAngles; const dx = Math.round(Math.cos(angle) * d); @@ -316,7 +316,12 @@ export function fastLocalInpaint( let valid = true; for (let y = ymin; y <= ymax && valid; y++) { const sy = y + dy; - if (sy < 0 || sy >= h) { valid = false; break; } + if (sy < 0 || sy >= h) { + for (let x = xmin; x <= xmax; x++) { + if (hole[y * w + x]) { valid = false; break; } + } + continue; + } const row = y * w; const sRow = sy * w; for (let x = xmin; x <= xmax; x++) { @@ -367,12 +372,15 @@ export function fastLocalInpaint( } } - // 4. Fill donor colors - const donorR = new Uint8Array(total); - const donorG = new Uint8Array(total); - const donorB = new Uint8Array(total); + const out = new Uint8Array(rgba.length); + out.set(rgba); if (foundDonor) { + // 4A. SEAMLESS POISSON BLENDING WITH CLEAN PHOTOGRAPHIC DONOR + const donorR = new Uint8Array(total); + const donorG = new Uint8Array(total); + const donorB = new Uint8Array(total); + for (let y = ymin; y <= ymax; y++) { const row = y * w; const sRow = (y + bestDy) * w; @@ -386,151 +394,191 @@ export function fastLocalInpaint( donorB[idx] = rgba[sp + 2]; } } - } else { - // Robust fallback: sample nearest clean boundary pixel - const fallbackBand = band.length > 0 ? band : [0]; + + const diffR = new Float32Array(total); + const diffG = new Float32Array(total); + const diffB = new Float32Array(total); + + let avgEdgeDiffR = 0, avgEdgeDiffG = 0, avgEdgeDiffB = 0; + let edgeCount = 0; + + for (let i = 0; i < holeBoundary.length; i++) { + const idx = holeBoundary[i]; + const x = idx % w; + const y = (idx / w) | 0; + + let cR = 0, cG = 0, cB = 0, cN = 0; + if (x > 0 && clean[idx - 1]) { const p = (idx - 1) << 2; cR += rgba[p]; cG += rgba[p + 1]; cB += rgba[p + 2]; cN++; } + if (x < w - 1 && clean[idx + 1]) { const p = (idx + 1) << 2; cR += rgba[p]; cG += rgba[p + 1]; cB += rgba[p + 2]; cN++; } + if (y > 0 && clean[idx - w]) { const p = (idx - w) << 2; cR += rgba[p]; cG += rgba[p + 1]; cB += rgba[p + 2]; cN++; } + if (y < h - 1 && clean[idx + w]) { const p = (idx + w) << 2; cR += rgba[p]; cG += rgba[p + 1]; cB += rgba[p + 2]; cN++; } + + if (cN > 0) { + const er = (cR / cN) - donorR[idx]; + const eg = (cG / cN) - donorG[idx]; + const eb = (cB / cN) - donorB[idx]; + diffR[idx] = er; diffG[idx] = eg; diffB[idx] = eb; + avgEdgeDiffR += er; avgEdgeDiffG += eg; avgEdgeDiffB += eb; + edgeCount++; + } + } + + if (edgeCount > 0) { + avgEdgeDiffR /= edgeCount; + avgEdgeDiffG /= edgeCount; + avgEdgeDiffB /= edgeCount; + } + for (let y = ymin; y <= ymax; y++) { const row = y * w; for (let x = xmin; x <= xmax; x++) { const idx = row + x; - if (!hole[idx]) continue; - let cIdx = fallbackBand[0]; - let cDist = Infinity; - for (let i = 0; i < fallbackBand.length; i++) { - const bx = fallbackBand[i] % w; - const by = (fallbackBand[i] / w) | 0; - const dist = (x - bx) * (x - bx) + (y - by) * (y - by); - if (dist < cDist) { cDist = dist; cIdx = fallbackBand[i]; } + if (hole[idx] && diffR[idx] === 0 && diffG[idx] === 0 && diffB[idx] === 0) { + diffR[idx] = avgEdgeDiffR; + diffG[idx] = avgEdgeDiffG; + diffB[idx] = avgEdgeDiffB; } - const cp = cIdx << 2; - donorR[idx] = rgba[cp]; - donorG[idx] = rgba[cp + 1]; - donorB[idx] = rgba[cp + 2]; } } - } - // 5. Difference Field Setup for Poisson Seamless Blending - const diffR = new Float32Array(total); - const diffG = new Float32Array(total); - const diffB = new Float32Array(total); + const omega = 1.75; + const oneMinusOmega = 1.0 - omega; + const iterations = 28; - let avgEdgeDiffR = 0, avgEdgeDiffG = 0, avgEdgeDiffB = 0; - let edgeCount = 0; + for (let it = 0; it < iterations; it++) { + for (let y = ymin; y <= ymax; y++) { + const row = y * w; + for (let x = xmin; x <= xmax; x++) { + const idx = row + x; + if (!hole[idx]) continue; - for (let i = 0; i < holeBoundary.length; i++) { - const idx = holeBoundary[i]; - const x = idx % w; - const y = (idx / w) | 0; + const rR = x < w - 1 ? diffR[idx + 1] : (x > 0 ? diffR[idx - 1] : diffR[idx]); + const rL = x > 0 ? diffR[idx - 1] : (x < w - 1 ? diffR[idx + 1] : diffR[idx]); + const rD = y < h - 1 ? diffR[idx + w] : (y > 0 ? diffR[idx - w] : diffR[idx]); + const rU = y > 0 ? diffR[idx - w] : (y < h - 1 ? diffR[idx + w] : diffR[idx]); - let cR = 0, cG = 0, cB = 0, cN = 0; - if (x > 0 && clean[idx - 1]) { - const p = (idx - 1) << 2; - cR += rgba[p]; cG += rgba[p + 1]; cB += rgba[p + 2]; cN++; - } - if (x < w - 1 && clean[idx + 1]) { - const p = (idx + 1) << 2; - cR += rgba[p]; cG += rgba[p + 1]; cB += rgba[p + 2]; cN++; - } - if (y > 0 && clean[idx - w]) { - const p = (idx - w) << 2; - cR += rgba[p]; cG += rgba[p + 1]; cB += rgba[p + 2]; cN++; - } - if (y < h - 1 && clean[idx + w]) { - const p = (idx + w) << 2; - cR += rgba[p]; cG += rgba[p + 1]; cB += rgba[p + 2]; cN++; - } + diffR[idx] = oneMinusOmega * diffR[idx] + omega * (rR + rL + rD + rU) * 0.25; - if (cN > 0) { - const er = (cR / cN) - donorR[idx]; - const eg = (cG / cN) - donorG[idx]; - const eb = (cB / cN) - donorB[idx]; - diffR[idx] = er; - diffG[idx] = eg; - diffB[idx] = eb; - avgEdgeDiffR += er; - avgEdgeDiffG += eg; - avgEdgeDiffB += eb; - edgeCount++; - } - } + const gR = x < w - 1 ? diffG[idx + 1] : (x > 0 ? diffG[idx - 1] : diffG[idx]); + const gL = x > 0 ? diffG[idx - 1] : (x < w - 1 ? diffG[idx + 1] : diffG[idx]); + const gD = y < h - 1 ? diffG[idx + w] : (y > 0 ? diffG[idx - w] : diffG[idx]); + const gU = y > 0 ? diffG[idx - w] : (y < h - 1 ? diffG[idx + w] : diffG[idx]); - if (edgeCount > 0) { - avgEdgeDiffR /= edgeCount; - avgEdgeDiffG /= edgeCount; - avgEdgeDiffB /= edgeCount; - } + diffG[idx] = oneMinusOmega * diffG[idx] + omega * (gR + gL + gD + gU) * 0.25; - for (let y = ymin; y <= ymax; y++) { - const row = y * w; - for (let x = xmin; x <= xmax; x++) { - const idx = row + x; - if (hole[idx] && diffR[idx] === 0 && diffG[idx] === 0 && diffB[idx] === 0) { - diffR[idx] = avgEdgeDiffR; - diffG[idx] = avgEdgeDiffG; - diffB[idx] = avgEdgeDiffB; + const bR = x < w - 1 ? diffB[idx + 1] : (x > 0 ? diffB[idx - 1] : diffB[idx]); + const bL = x > 0 ? diffB[idx - 1] : (x < w - 1 ? diffB[idx + 1] : diffB[idx]); + const bD = y < h - 1 ? diffB[idx + w] : (y > 0 ? diffB[idx - w] : diffB[idx]); + const bU = y > 0 ? diffB[idx - w] : (y < h - 1 ? diffB[idx + w] : diffB[idx]); + + diffB[idx] = oneMinusOmega * diffB[idx] + omega * (bR + bL + bD + bU) * 0.25; + } } } - } - // 6. Successive Over-Relaxation (SOR) solver for Laplace equation ∇² E = 0 - const omega = 1.75; - const oneMinusOmega = 1.0 - omega; - const iterations = 28; - - for (let it = 0; it < iterations; it++) { for (let y = ymin; y <= ymax; y++) { const row = y * w; for (let x = xmin; x <= xmax; x++) { const idx = row + x; if (!hole[idx]) continue; - - const rR = x < w - 1 ? diffR[idx + 1] : (x > 0 ? diffR[idx - 1] : diffR[idx]); - const rL = x > 0 ? diffR[idx - 1] : (x < w - 1 ? diffR[idx + 1] : diffR[idx]); - const rD = y < h - 1 ? diffR[idx + w] : (y > 0 ? diffR[idx - w] : diffR[idx]); - const rU = y > 0 ? diffR[idx - w] : (y < h - 1 ? diffR[idx + w] : diffR[idx]); - - const avgR = (rR + rL + rD + rU) * 0.25; - diffR[idx] = oneMinusOmega * diffR[idx] + omega * avgR; - - const gR = x < w - 1 ? diffG[idx + 1] : (x > 0 ? diffG[idx - 1] : diffG[idx]); - const gL = x > 0 ? diffG[idx - 1] : (x < w - 1 ? diffG[idx + 1] : diffG[idx]); - const gD = y < h - 1 ? diffG[idx + w] : (y > 0 ? diffG[idx - w] : diffG[idx]); - const gU = y > 0 ? diffG[idx - w] : (y < h - 1 ? diffG[idx + w] : diffG[idx]); - - const avgG = (gR + gL + gD + gU) * 0.25; - diffG[idx] = oneMinusOmega * diffG[idx] + omega * avgG; - - const bR = x < w - 1 ? diffB[idx + 1] : (x > 0 ? diffB[idx - 1] : diffB[idx]); - const bL = x > 0 ? diffB[idx - 1] : (x < w - 1 ? diffB[idx + 1] : diffB[idx]); - const bD = y < h - 1 ? diffB[idx + w] : (y > 0 ? diffB[idx - w] : diffB[idx]); - const bU = y > 0 ? diffB[idx - w] : (y < h - 1 ? diffB[idx + w] : diffB[idx]); - - const avgB = (bR + bL + bD + bU) * 0.25; - diffB[idx] = oneMinusOmega * diffB[idx] + omega * avgB; + const p = idx << 2; + const r = Math.round(donorR[idx] + diffR[idx]); + const g = Math.round(donorG[idx] + diffG[idx]); + const b = Math.round(donorB[idx] + diffB[idx]); + out[p] = r < 0 ? 0 : r > 255 ? 255 : r; + out[p + 1] = g < 0 ? 0 : g > 255 ? 255 : g; + out[p + 2] = b < 0 ? 0 : b > 255 ? 255 : b; + out[p + 3] = 255; } } - } + } else { + // 4B. ROBUST DIRICHLET HARMONIC RELAXATION FALLBACK (smooth, continuous, zero Voronoi facets) + const baseR = new Float32Array(total); + const baseG = new Float32Array(total); + const baseB = new Float32Array(total); - // 7. Assemble final healed plate - const out = new Uint8Array(rgba.length); - out.set(rgba); + for (let i = 0; i < total; i++) { + if (clean[i]) { + const p = i << 2; + baseR[i] = rgba[p]; + baseG[i] = rgba[p + 1]; + baseB[i] = rgba[p + 2]; + } + } - for (let y = ymin; y <= ymax; y++) { - const row = y * w; - for (let x = xmin; x <= xmax; x++) { - const idx = row + x; - if (!hole[idx]) continue; + for (let y = ymin; y <= ymax; y++) { + const row = y * w; + for (let x = xmin; x <= xmax; x++) { + const idx = row + x; + if (clean[idx]) continue; - const p = idx << 2; - const r = Math.round(donorR[idx] + diffR[idx]); - const g = Math.round(donorG[idx] + diffG[idx]); - const b = Math.round(donorB[idx] + diffB[idx]); + let xl = x, xr = x, yt = y, yb = y; + while (xl >= 0 && !clean[row + xl]) xl--; + while (xr < w && !clean[row + xr]) xr++; + while (yt >= 0 && !clean[yt * w + x]) yt--; + while (yb < h && !clean[yb * w + x]) yb++; - out[p] = r < 0 ? 0 : r > 255 ? 255 : r; - out[p + 1] = g < 0 ? 0 : g > 255 ? 255 : g; - out[p + 2] = b < 0 ? 0 : b > 255 ? 255 : b; - out[p + 3] = 255; + let sumR = 0, sumG = 0, sumB = 0, sumW = 0; + if (xl >= 0) { const d = x - xl; const wt = 1.0 / (d * d); const p = (row + xl) << 2; sumR += rgba[p] * wt; sumG += rgba[p + 1] * wt; sumB += rgba[p + 2] * wt; sumW += wt; } + if (xr < w) { const d = xr - x; const wt = 1.0 / (d * d); const p = (row + xr) << 2; sumR += rgba[p] * wt; sumG += rgba[p + 1] * wt; sumB += rgba[p + 2] * wt; sumW += wt; } + if (yt >= 0) { const d = y - yt; const wt = 1.0 / (d * d); const p = (yt * w + x) << 2; sumR += rgba[p] * wt; sumG += rgba[p + 1] * wt; sumB += rgba[p + 2] * wt; sumW += wt; } + if (yb < h) { const d = yb - y; const wt = 1.0 / (d * d); const p = (yb * w + x) << 2; sumR += rgba[p] * wt; sumG += rgba[p + 1] * wt; sumB += rgba[p + 2] * wt; sumW += wt; } + + if (sumW > 0) { + baseR[idx] = sumR / sumW; + baseG[idx] = sumG / sumW; + baseB[idx] = sumB / sumW; + } + } + } + + const omega = 1.75; + const oneMinusOmega = 1.0 - omega; + for (let it = 0; it < 32; it++) { + for (let y = ymin; y <= ymax; y++) { + const row = y * w; + for (let x = xmin; x <= xmax; x++) { + const idx = row + x; + if (clean[idx]) continue; + + const rR = x < w - 1 ? baseR[idx + 1] : (x > 0 ? baseR[idx - 1] : baseR[idx]); + const rL = x > 0 ? baseR[idx - 1] : (x < w - 1 ? baseR[idx + 1] : baseR[idx]); + const rD = y < h - 1 ? baseR[idx + w] : (y > 0 ? baseR[idx - w] : baseR[idx]); + const rU = y > 0 ? baseR[idx - w] : (y < h - 1 ? baseR[idx + w] : baseR[idx]); + + baseR[idx] = oneMinusOmega * baseR[idx] + omega * (rR + rL + rD + rU) * 0.25; + + const gR = x < w - 1 ? baseG[idx + 1] : (x > 0 ? baseG[idx - 1] : baseG[idx]); + const gL = x > 0 ? baseG[idx - 1] : (x < w - 1 ? baseG[idx + 1] : baseG[idx]); + const gD = y < h - 1 ? baseG[idx + w] : (y > 0 ? baseG[idx - w] : baseG[idx]); + const gU = y > 0 ? baseG[idx - w] : (y < h - 1 ? baseG[idx + w] : baseG[idx]); + + baseG[idx] = oneMinusOmega * baseG[idx] + omega * (gR + gL + gD + gU) * 0.25; + + const bR = x < w - 1 ? baseB[idx + 1] : (x > 0 ? baseB[idx - 1] : baseB[idx]); + const bL = x > 0 ? baseB[idx - 1] : (x < w - 1 ? baseB[idx + 1] : baseB[idx]); + const bD = y < h - 1 ? baseB[idx + w] : (y > 0 ? baseB[idx - w] : baseB[idx]); + const bU = y > 0 ? baseB[idx - w] : (y < h - 1 ? baseB[idx + w] : baseB[idx]); + + baseB[idx] = oneMinusOmega * baseB[idx] + omega * (bR + bL + bD + bU) * 0.25; + } + } + } + + for (let y = ymin; y <= ymax; y++) { + const row = y * w; + for (let x = xmin; x <= xmax; x++) { + const idx = row + x; + if (!hole[idx]) continue; + const p = idx << 2; + const r = Math.round(baseR[idx]); + const g = Math.round(baseG[idx]); + const b = Math.round(baseB[idx]); + out[p] = r < 0 ? 0 : r > 255 ? 255 : r; + out[p + 1] = g < 0 ? 0 : g > 255 ? 255 : g; + out[p + 2] = b < 0 ? 0 : b > 255 ? 255 : b; + out[p + 3] = 255; + } } } @@ -693,7 +741,7 @@ export async function inpaintCrop( const maskB64 = uint8ToBase64(mask); const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 8500); + const timeoutId = setTimeout(() => controller.abort(), 15000); const res = await fetch('/api/inpaint', { method: 'POST',