From bbc64118a55dc77127ae9d7a4cbaed01cf568ef0 Mon Sep 17 00:00:00 2001 From: 3dtours Date: Sat, 17 Oct 2026 11:01:00 +0700 Subject: [PATCH] fix(heal): upgrade inpaint architecture to hybrid 2-tier LaMa ONNX and fix starburst artifacts - backend: preserve aspect ratio with edge-replicated square padding into 512x512 LaMa model - backend: fix inverted mask threshold (>0.85*plane) and prune CUDA/TensorRT binaries for slim image - frontend: enable server AI by default and route masks >3.5% or edge-crossing to LaMa Tier 2 - frontend: enforce 60%-80% context padding (min 64px) aligned to multiples of 8 - frontend: add 3-5px binary mask dilation and high-frequency sensor grain matching - frontend: anchor deletion badge to topmost point of brush stroke - build & export: updated docker containers and recipescam-images.tar.gz (357MB) --- docker/backend/Dockerfile | 8 + docker/backend/package.json | 3 +- docker/backend/src/inpaintService.ts | 121 ++- docker/backend/test/inpaint-onnx.mjs | 92 ++ docker/frontend/scripts/heal-check.mjs | 7 +- docker/frontend/scripts/heal-route-check.mjs | 253 +++++ .../frontend/scripts/heal-texture-check.mjs | 91 ++ docker/frontend/src/App.tsx | 2 +- docker/frontend/src/engine/exportEngine.ts | 27 +- docker/frontend/src/engine/inpaint.ts | 988 +++++++++++------- docker/frontend/src/styles/app.css | 46 + docker/frontend/src/styles/tokens.css | 5 + docker/frontend/src/ui/ImageStage.tsx | 12 +- docker/recipescam-images.tar.gz.sha256 | 2 +- 14 files changed, 1242 insertions(+), 415 deletions(-) create mode 100644 docker/backend/test/inpaint-onnx.mjs create mode 100644 docker/frontend/scripts/heal-route-check.mjs create mode 100644 docker/frontend/scripts/heal-texture-check.mjs diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index 59cf3d9..ed1e419 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -7,6 +7,14 @@ RUN npm ci COPY tsconfig.json ./ COPY src ./src RUN npm run build && npm prune --omit=dev +# onnxruntime-node ships every platform's runtime, and 260MB of it is the CUDA / +# TensorRT providers this container (CPU inference, no GPU) never loads. The +# bindings for the other two operating systems go the same way; both Linux +# architectures stay so the image can still be built for an ARM host. +RUN rm -rf node_modules/onnxruntime-node/bin/napi-v6/darwin \ + node_modules/onnxruntime-node/bin/napi-v6/win32 \ + node_modules/onnxruntime-node/bin/napi-v6/linux/x64/libonnxruntime_providers_cuda.so \ + node_modules/onnxruntime-node/bin/napi-v6/linux/x64/libonnxruntime_providers_tensorrt.so FROM node:22-slim ENV NODE_ENV=production \ diff --git a/docker/backend/package.json b/docker/backend/package.json index a6e331b..c957b08 100644 --- a/docker/backend/package.json +++ b/docker/backend/package.json @@ -9,7 +9,8 @@ "start": "node dist/server.js", "dev": "tsx watch src/server.ts", "test": "node test/security.mjs", - "test:immich": "node test/immich.mjs" + "test:immich": "node test/immich.mjs", + "test:inpaint": "node test/inpaint-onnx.mjs" }, "dependencies": { "better-sqlite3": "^12.11.1", diff --git a/docker/backend/src/inpaintService.ts b/docker/backend/src/inpaintService.ts index ee795d3..e34c144 100644 --- a/docker/backend/src/inpaintService.ts +++ b/docker/backend/src/inpaintService.ts @@ -1,8 +1,13 @@ import * as ort from 'onnxruntime-node'; import { existsSync } from 'node:fs'; +import { cpus } from 'node:os'; const MODEL_PATH = process.env.INPAINT_MODEL_PATH || '/models/lama_fp32.onnx'; +// What the graph will take spatially, learned on the first call: a resolution-robust export runs +// at the crop's own size, a pinned one reports the square it was traced at and we remember it. +let fixedSide: number | null = null; + let sessionPromise: Promise | null = null; export async function getInpaintSession(): Promise { @@ -13,8 +18,6 @@ export async function getInpaintSession(): Promise '/app/models/lama_fp32.onnx', '/models/lama_fp32.onnx', './models/lama_fp32.onnx', - '/home/locpham/RecipesCam/docker/frontend/public/models/lama_fp32.onnx', - '../frontend/public/models/lama_fp32.onnx', ]; let foundPath: string | null = null; for (const p of candidates) { @@ -33,6 +36,9 @@ export async function getInpaintSession(): Promise const session = await ort.InferenceSession.create(foundPath, { executionProviders: ['cpu'], graphOptimizationLevel: 'all', + // Measured on the dev box's 4-core i3: 4 threads 3.0s, default (8) 3.8s, 2 threads 4.0s. + // Half the cores is the knee, and it leaves the rest of the API its share on a NAS. + intraOpNumThreads: Math.max(1, Math.min(4, Math.floor(cpus().length / 2))), }); console.log('[inpaintService] Model loaded in ' + (Date.now() - t0) + 'ms'); console.log('[inpaintService] Inputs:', session.inputNames, 'Outputs:', session.outputNames); @@ -106,38 +112,113 @@ export async function runServerInpaint( rawRgb[i] = srcBytes[i] / 255.0; } + // Any non-zero byte is the hole: the model's convention is a binary white-on-black mask, and + // the caller's white may arrive as 255, 1 or anything between. A mask that reads as all-zero + // here is not a repair at all — LaMa answers with the image it was handed — so the byte that + // says "hole" is worth accepting in every spelling. + // Inversion check: only flip if > 85% of the patch is marked as hole (suggesting inverted alpha mask) const rawMask = new Float32Array(plane); + let holes = 0; for (let i = 0; i < plane; i++) { - rawMask[i] = maskBytes[i] > 128 ? 1.0 : 0.0; + if (maskBytes[i] !== 0) { + rawMask[i] = 1.0; + holes++; + } } - - // Bilinear resize to 512x512 for LaMa - const inImage512 = resizeBilinear(rawRgb, w, h, 512, 512, 3); - const inMask512 = resizeBilinear(rawMask, w, h, 512, 512, 1); - for (let i = 0; i < 512 * 512; i++) { - inMask512[i] = inMask512[i] > 0.15 ? 1.0 : 0.0; + if (holes > 0.85 * plane) { + console.warn(`[inpaintService] mask is ${Math.round((100 * holes) / plane)}% of the crop: read as inverted, flipping`); + for (let i = 0; i < plane; i++) rawMask[i] = rawMask[i] > 0 ? 0 : 1; } + console.log(`[inpaintService] mask ${w}x${h}: ${holes} hole px of ${plane} (${((100 * holes) / plane).toFixed(1)}%)`); const imageInputName = session.inputNames.includes('image') ? 'image' : session.inputNames[0]; const maskInputName = session.inputNames.includes('mask') ? 'mask' : session.inputNames[1]; - const out = await session.run({ - [imageInputName]: new ort.Tensor('float32', inImage512, [1, 3, 512, 512]), - [maskInputName]: new ort.Tensor('float32', inMask512, [1, 1, 512, 512]), - }); + // Uniform aspect-ratio preserving resize and padding to fit the side the graph accepts (e.g. 512). + // Preserving aspect ratio prevents anamorphic squashing/stretching that causes radial starburst + // and Fourier spectral leakage artifacts. + const runAt = async (side: number): Promise => { + const scale = Math.min(side / w, side / h); + const scaledW = Math.max(1, Math.min(side, Math.round(w * scale))); + const scaledH = Math.max(1, Math.min(side, Math.round(h * scale))); - const outTensor = out[session.outputNames[0]]; - const out512 = outTensor.data as Float32Array; + const resizedImg = resizeBilinear(rawRgb, w, h, scaledW, scaledH, 3); + const resizedMask = resizeBilinear(rawMask, w, h, scaledW, scaledH, 1); - // Resize output back to w x h - const outSw = resizeBilinear(out512, 512, 512, w, h, 3); + // Pad into side x side buffer with edge replication for image and 0 for mask + const inImage = new Float32Array(3 * side * side); + const inMask = new Float32Array(side * side); + + for (let y = 0; y < side; y++) { + const sy = Math.min(scaledH - 1, y); + for (let x = 0; x < side; x++) { + const sx = Math.min(scaledW - 1, x); + const dstIdx = y * side + x; + const srcIdx = sy * scaledW + sx; + + inImage[dstIdx] = resizedImg[srcIdx]; + inImage[side * side + dstIdx] = resizedImg[scaledW * scaledH + srcIdx]; + inImage[2 * side * side + dstIdx] = resizedImg[2 * scaledW * scaledH + srcIdx]; + + if (y < scaledH && x < scaledW) { + inMask[dstIdx] = resizedMask[srcIdx] > 0.15 ? 1.0 : 0.0; + } + } + } + + const out = await session.run({ + [imageInputName]: new ort.Tensor('float32', inImage, [1, 3, side, side]), + [maskInputName]: new ort.Tensor('float32', inMask, [1, 1, side, side]), + }); + + const outTensor = out[session.outputNames[0]]; + const outData = outTensor.data as Float32Array; + + // Extract the scaledW x scaledH subregion + const croppedOut = new Float32Array(3 * scaledW * scaledH); + for (let y = 0; y < scaledH; y++) { + for (let x = 0; x < scaledW; x++) { + const srcIdx = y * side + x; + const dstIdx = y * scaledW + x; + croppedOut[dstIdx] = outData[srcIdx]; + croppedOut[scaledW * scaledH + dstIdx] = outData[side * side + srcIdx]; + croppedOut[2 * scaledW * scaledH + dstIdx] = outData[2 * side * side + srcIdx]; + } + } + + // Bilinear resize back to crop w x h + return resizeBilinear(croppedOut, scaledW, scaledH, w, h, 3); + }; + + let outSw: Float32Array; + if (fixedSide) { + outSw = await runAt(fixedSide); + } else { + const side = Math.max(64, Math.min(768, Math.ceil(Math.max(w, h) / 8) * 8)); + try { + outSw = await runAt(side); + } catch (err) { + const expected = /Expected: (\d+)/.exec(String((err as Error)?.message || err)); + if (!expected) throw err; + fixedSide = Number(expected[1]); + outSw = await runAt(fixedSide); + } + } + + // Adaptive scale: detect if model output is [0..1] vs [0..255] + let maxVal = -Infinity; + const testLen = Math.min(500, outSw.length); + for (let i = 0; i < testLen; i++) { + if (outSw[i] > maxVal) maxVal = outSw[i]; + } + const mult = maxVal <= 2.5 && maxVal > 0 ? 255.0 : 1.0; const patch = new Uint8Array(4 * plane); const plane2 = 2 * plane; for (let i = 0; i < plane; i++) { - const r = Math.round(outSw[i]); - const g = Math.round(outSw[plane + i]); - const b = Math.round(outSw[plane2 + i]); + const r = Math.round(outSw[i] * mult); + const g = Math.round(outSw[plane + i] * mult); + const b = Math.round(outSw[plane2 + i] * mult); patch[i * 4] = r < 0 ? 0 : r > 255 ? 255 : r; patch[i * 4 + 1] = g < 0 ? 0 : g > 255 ? 255 : g; diff --git a/docker/backend/test/inpaint-onnx.mjs b/docker/backend/test/inpaint-onnx.mjs new file mode 100644 index 0000000..12b0304 --- /dev/null +++ b/docker/backend/test/inpaint-onnx.mjs @@ -0,0 +1,92 @@ +// The server's half of HEAL: lama_fp32.onnx behind ONNX Runtime, and the one thing about it the +// client cannot know — which spatial sizes the graph will actually take. The shipped export is +// pinned to 512x512 (its own traced square), while a resolution-robust export takes the crop's +// own size; either way the endpoint must answer at the size it was asked, so the service tries +// the crop first, learns the square from the refusal, and goes straight there afterwards. +// +// What is asserted: a 256x256 crop comes back as a 256x256 RGBA patch with an opaque alpha (the +// refused call is retried, not surfaced), the hole is filled with something coloured like the +// ground around it rather than black or noise, and the pixels outside it are left alone by the +// resize round-trip. +// +// node test/inpaint-onnx.mjs (from docker/backend/) +import assert from 'node:assert/strict'; +import { existsSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import ts from 'typescript'; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); +const MODEL = join(ROOT, 'models', 'lama_fp32.onnx'); +if (!existsSync(MODEL)) { + console.log(`SKIP no model at ${MODEL}`); + process.exit(0); +} + +// Transpiled next to node_modules, not in /tmp: the service imports onnxruntime-node. +const dir = mkdtempSync(join(ROOT, 'node_modules', '.inpaint-onnx-')); +writeFileSync( + join(dir, 'inpaintService.mjs'), + ts.transpileModule(readFileSync(join(ROOT, 'src', 'inpaintService.ts'), 'utf8'), { + compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2022 }, + }).outputText +); +process.env.INPAINT_MODEL_PATH = MODEL; +const { runServerInpaint } = await import(pathToFileURL(join(dir, 'inpaintService.mjs')).href); + +// A 256x256 crop of a sky-and-foliage sort of scene, with a dab over one corner of it. The dab +// is a different colour from everything around it, so a patch that kept it is a patch that failed. +const W = 256, H = 256, PLANE = W * H; +const src = new Uint8Array(3 * PLANE); +const mask = new Uint8Array(PLANE); +const truth = (x, y) => 120 + 40 * Math.sin(x * 0.03) + 25 * Math.cos(y * 0.05); +for (let y = 0; y < H; y++) { + for (let x = 0; x < W; x++) { + const i = y * W + x; + const v = truth(x, y) + 8 * Math.sin(x * 0.7 + y * 0.9); + src[i] = Math.max(0, Math.min(255, v)); + src[PLANE + i] = Math.max(0, Math.min(255, v * 0.95 + 6)); + src[2 * PLANE + i] = Math.max(0, Math.min(255, v * 0.8 + 20)); + if (x > 90 && x < 130 && y > 90 && y < 130) { + mask[i] = 255; + src[i] = 250; + src[PLANE + i] = 10; + src[2 * PLANE + i] = 250; // magenta mark, nothing like the ground + } + } +} + +const t0 = Date.now(); +const patch = await runServerInpaint(src, mask, W, H); +const ms = Date.now() - t0; +assert.ok(patch, 'the model must answer at 256x256 even though the export is pinned to 512'); + +assert.equal(patch.length, 4 * PLANE, 'RGBA patch at the size asked for'); +let opaque = true; +for (let i = 3; i < patch.length; i += 4) if (patch[i] !== 255) opaque = false; +assert.ok(opaque, 'every pixel of the patch is opaque'); + +// The mark is gone: inside the hole the patch has to look like the ground, not like the mark and +// not like nothing at all. +let holeErr = 0, holeN = 0, outsideErr = 0, outsideN = 0; +for (let y = 0; y < H; y++) { + for (let x = 0; x < W; x++) { + const i = y * W + x, p = i * 4; + const want = truth(x, y); + const got = 0.299 * patch[p] + 0.587 * patch[p + 1] + 0.114 * patch[p + 2]; + if (mask[i]) { + holeErr += Math.abs(got - want); + holeN++; + } else { + const srcLuma = 0.299 * src[i] + 0.587 * src[PLANE + i] + 0.114 * src[2 * PLANE + i]; + outsideErr += Math.abs(got - srcLuma); + outsideN++; + } + } +} +const holeMae = holeErr / Math.max(1, holeN); +const outsideMae = outsideErr / Math.max(1, outsideN); +console.log(`patch ${W}x${H} in ${ms}ms, hole MAE ${holeMae.toFixed(2)}, outside MAE ${outsideMae.toFixed(2)}`); +assert.ok(holeMae < 25, `hole MAE ${holeMae.toFixed(2)} — the mark must be gone, not kept`); +assert.ok(outsideMae < 6, `outside MAE ${outsideMae.toFixed(2)} — clean pixels must survive the round trip`); +console.log('PASS inpaint-onnx'); diff --git a/docker/frontend/scripts/heal-check.mjs b/docker/frontend/scripts/heal-check.mjs index 70b2547..8c16597 100644 --- a/docker/frontend/scripts/heal-check.mjs +++ b/docker/frontend/scripts/heal-check.mjs @@ -59,15 +59,16 @@ assert.match(engine, /drawImageRect\(cutImage, box, onto, /, 'the fill is drawn // answer is written into the fill surface over the box, and the mask it is // handed is built from the brush's own circles. What it is NOT is measured // here — a working model is invisible to the numbers below, which are the -// fallback's. The polarity is MI-GAN's, 255 KEEP and 0 HOLE, and it is the one +// fallback's. The polarity is the model's, 255 HOLE and 0 KEEP, and it is the one // thing that, got backwards, repaints the frame and leaves the hole (measured in // engine/inpaint.ts; the probe is gone). const inpaint = readFileSync(new URL('../src/engine/inpaint.ts', import.meta.url), 'utf8'); -assert.match(engine, /await inpaintCrop\(cropPixels, mw, mh, hole\)/, 'exportEngine does not ask the model for the masked crop'); +assert.match(engine, /await inpaintCrop\(cropPixels, mw, mh, hole, brushArea \/ \(width \* height\)\)/, 'exportEngine does not ask the model for the masked crop, measured against the frame'); assert.match(engine, /patchSurface\.getCanvas\(\)\.writePixels\(patch, mw, mh, 0, 0\)/, 'the model answer is never written into the fill surface'); assert.match(engine, /new Uint8Array\(mw \* mh\)/, 'the mask handed to the model is not built at the crop\'s own resolution'); assert.match(engine, /hole\[row \+ x\] = 1/, 'the mask handed to the model is not the brush\'s own circles'); -assert.match(inpaint, /fill\(255\)/, 'the model mask no longer starts at KEEP'); +assert.match(inpaint, /const mask = new Uint8Array\(plane\)/, 'the model mask no longer starts at KEEP'); +assert.match(inpaint, /mask\[d\] = 255/, 'the brush\'s circles are no longer marked as holes in the model mask'); assert.match(inpaint, /INPAINT_MAX_SIDE/, 'the crop handed to the model is no longer bounded in size'); assert.match(inpaint, /inpaintCropSize/, 'nothing tells the renderer the resolution the model will run at'); diff --git a/docker/frontend/scripts/heal-route-check.mjs b/docker/frontend/scripts/heal-route-check.mjs new file mode 100644 index 0000000..4ee5c51 --- /dev/null +++ b/docker/frontend/scripts/heal-route-check.mjs @@ -0,0 +1,253 @@ +// The hybrid's three client-side rules, measured on synthetic frames whose answer is known: +// +// * the router — which of the two engines a mask goes to (a speck of dust on open ground is the +// fast engine's job and costs nothing; an object, or a mark that sits across a real edge, +// goes to the model), +// * the mask the model is given — grown a few pixels over the stroke so the object's shadow +// goes with it, and only over the stroke, because the brush is not the wound, +// * the grain the model leaves behind — a model patch is a clean surface, and a clean surface +// in a photograph of sensor noise reads as a smudge, so it is given back the ground's own +// grain and nothing where the ground has none. +// +// node scripts/heal-route-check.mjs +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import ts from 'typescript'; + +const SRC = new URL('../src/engine/inpaint.ts', import.meta.url); +const dir = mkdtempSync(join(tmpdir(), 'heal-route-')); +writeFileSync( + join(dir, 'inpaint.mjs'), + ts.transpileModule(readFileSync(SRC, 'utf8'), { + compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2022 }, + }).outputText +); +const { aiTierNeeded, dilateMask, matchGrain, AI_MAX_MASK_AREA, fastLocalInpaint, localFillWasFlat } = await import( + pathToFileURL(join(dir, 'inpaint.mjs')).href +); + +const W = 512, H = 512; +const luma = (v) => Math.max(0, Math.min(255, v)); + +// A frame builder: `at(x, y)` says what the ground looks like, and the hole is a disc on it. +function frame(at, { cx = 256, cy = 256, r = 12, noise = 0, seed = 7 } = {}) { + const rgba = new Uint8Array(W * H * 4); + const hole = new Uint8Array(W * H); + let s = seed; + const rnd = () => ((s = (s * 1103515245 + 12345) & 0x7fffffff) / 0x7fffffff) - 0.5; + for (let y = 0; y < H; y++) { + for (let x = 0; x < W; x++) { + const i = y * W + x, p = i * 4; + const v = luma(at(x, y) + (noise ? rnd() * noise : 0)); + rgba[p] = rgba[p + 1] = rgba[p + 2] = v; + rgba[p + 3] = 255; + if (r > 0 && (x - cx) ** 2 + (y - cy) ** 2 <= r * r) hole[i] = 1; + } + } + return { rgba, hole }; +} + +const flat = () => 128; +const steps = (x) => (x < 128 || x > 380 ? 80 : 176); +const grain = () => 128; +const FRAME_AREA = W * H; // this frame is the whole picture for the routing tests + +// 1. A speck of dust on open ground: the fast engine, no server. +const speck = frame(flat, { r: 12 }); +assert.equal(aiTierNeeded(speck.rgba, W, H, speck.hole, (Math.PI * 12 * 12) / FRAME_AREA), false); + +// 2. Something the size of an object: the model. On this 512x512 frame a 60px radius is 4.3% of +// the area, which is the same order as a 5%-of-width brush on a 3:2 frame. +const object = frame(flat, { r: 60 }); +assert.equal(aiTierNeeded(object.rgba, W, H, object.hole, (Math.PI * 60 * 60) / FRAME_AREA), true); +assert.ok( + (Math.PI * 60 * 60) / FRAME_AREA > AI_MAX_MASK_AREA, + 'the object case must be over the 3.5% the chart draws' +); + +// 2b. The brush at its widest on open ground is still a speck by that measure: the fast engine. +const wideDab = frame(flat, { r: 26 }); +assert.equal(aiTierNeeded(wideDab.rgba, W, H, wideDab.hole, (Math.PI * 26 * 26) / FRAME_AREA), false); + +// 2c. With no frame to judge the area against, the window is what is left — and a real window is +// a tight one around its hole, so a mark that fills an eighth of it goes to the model. +const filled = frame(flat, { r: 100 }); +assert.equal(aiTierNeeded(filled.rgba, W, H, filled.hole), true); + +// 3. A speck, but one sitting across a real edge: the model, however small it is. The ground here +// is quiet (rim 1.88 of 255 either side of the line) so the area test would have kept it home; +// what sends it out is that the rim carries six times the window's own structure. +const onEdge = frame(steps, { cx: 128, cy: 256, r: 12 }); +assert.equal(aiTierNeeded(onEdge.rgba, W, H, onEdge.hole, (Math.PI * 12 * 12) / FRAME_AREA), true); + +// 3b. Busy ground — foliage, water ripples, a crowd — is where a copied patch shows up as a slab +// of somewhere else, so a speck there goes to the model too, even with no edge in sight. +const foliage = frame((x, y) => 120 + 50 * Math.abs(Math.sin(x * 0.21 + y * 0.13)) + 30 * Math.sin(x * 0.7 + y * 1.3), { r: 26 }); +assert.equal(aiTierNeeded(foliage.rgba, W, H, foliage.hole, (Math.PI * 26 * 26) / FRAME_AREA), true); + +// 3c. ...but ripples are not that busy, and the engine does answer them: a speck on water stays. +const water = frame((x, y) => 128 + 18 * Math.sin(x * 0.35 + y * 0.5) + 10 * Math.sin(x * 0.9 - y * 0.7), { r: 26 }); +assert.equal(aiTierNeeded(water.rgba, W, H, water.hole, (Math.PI * 26 * 26) / FRAME_AREA), false); + +// 4. A speck on ground that is nothing but sensor grain: the fast engine. Grain is not structure. +const noisy = frame(grain, { r: 12, noise: 14 }); +assert.equal(aiTierNeeded(noisy.rgba, W, H, noisy.hole, (Math.PI * 12 * 12) / FRAME_AREA), false); + +// ...and the same picture without the noise reads the same way, so noise is not what decided it. +const clean = frame(grain, { r: 12 }); +assert.equal(aiTierNeeded(clean.rgba, W, H, clean.hole, (Math.PI * 12 * 12) / FRAME_AREA), false); + +// 5. The mask handed to the model: grown by exactly the radius, and no further. The growth is a +// box, so a pixel may sit up to r diagonally away — but never outside the r-wide slab. +const grown = dilateMask(speck.hole, W, H, 3); +let grew = 0, shrank = 0, tooFar = 0; +for (let y = 0; y < H; y++) { + for (let x = 0; x < W; x++) { + const i = y * W + x; + if (!speck.hole[i] && grown[i]) { + grew++; + const d = Math.hypot(x - 256, y - 256); + if (d > 12 + 3 * Math.SQRT2 + 0.5) tooFar++; + } + if (speck.hole[i] && !grown[i]) shrank++; + } +} +assert.equal(shrank, 0, 'a grown mask still covers the stroke'); +assert.ok(grew > 200, `the mask must actually grow (a 3px ring on a 12px disc is ~250 pixels, got ${grew})`); +assert.equal(tooFar, 0, 'and not by more than the radius asked for'); + +// 6. Grain handed back. The patch is what a model returns — a smooth surface — and the ground +// under the same mask is the photo's own noise. After the call the patch's high-pass may no +// longer be a fraction of the ground's, and the pixels outside the mask are untouched. +const noisyGround = frame(grain, { r: 26, noise: 16 }); +const smoothPatch = new Uint8Array(4 * W * H); +for (let i = 0; i < W * H; i++) { + const p = i * 4; + smoothPatch[p] = smoothPatch[p + 1] = smoothPatch[p + 2] = 128; + smoothPatch[p + 3] = 255; +} +const before = smoothPatch.slice(); +const hp = (data, mask, inside, stride) => { + let sum = 0, sum2 = 0, n = 0; + for (let y = 1; y < H - 1; y += stride) { + for (let x = 1; x < W - 1; x += stride) { + const i = y * W + x; + if (inside ? !mask[i] : mask[i]) continue; + const m = (data[i - W - 1] + data[i - W] + data[i - W + 1] + data[i - 1] + data[i] + data[i + 1] + data[i + W - 1] + data[i + W] + data[i + W + 1]) / 9; + const d = data[i] - m; + sum += d; + sum2 += d * d; + n++; + } + } + return Math.sqrt(Math.max(0, sum2 / n - (sum / n) ** 2)); +}; +const groundPlane = new Uint8Array(W * H); +for (let i = 0; i < W * H; i++) groundPlane[i] = noisyGround.rgba[i * 4]; +// matchGrain reads the source the way the server is fed it: three planes, no alpha. +const src3 = new Uint8Array(3 * W * H); +for (let i = 0; i < W * H; i++) { + src3[i] = noisyGround.rgba[i * 4]; + src3[W * H + i] = noisyGround.rgba[i * 4 + 1]; + src3[2 * W * H + i] = noisyGround.rgba[i * 4 + 2]; +} +const patchPlane = new Uint8Array(W * H); +for (let i = 0; i < W * H; i++) patchPlane[i] = smoothPatch[i * 4]; +const groundSigma = hp(groundPlane, noisyGround.hole, false, 1); +const smoothSigma = hp(patchPlane, noisyGround.hole, true, 2); +const added = matchGrain(smoothPatch, src3, noisyGround.hole, W, H); +for (let i = 0; i < W * H; i++) patchPlane[i] = smoothPatch[i * 4]; +const nowSigma = hp(patchPlane, noisyGround.hole, true, 2); +let touchedOutside = 0; +for (let i = 0; i < W * H; i++) { + if (noisyGround.hole[i]) continue; + const p = i * 4; + if (smoothPatch[p] !== before[p]) touchedOutside++; +} +console.log( + `ground hp sigma ${groundSigma.toFixed(2)}, model patch ${smoothSigma.toFixed(2)}, ` + + `after grain ${nowSigma.toFixed(2)} (added ${added.toFixed(2)})` +); +assert.ok(groundSigma > 3, 'the synthetic ground has to be noisy for this to mean anything'); +assert.ok(nowSigma > groundSigma * 0.5, 'the model patch must end up carrying the ground grain'); +assert.ok(nowSigma < groundSigma * 2.2, 'and not more of it than the ground has'); +assert.equal(touchedOutside, 0, 'grain goes under the mask only'); + +// 7. ...and a patch that already has the ground's grain is not given a second helping. +const already = noisyGround.rgba.slice(); +const addedAgain = matchGrain(already, src3, noisyGround.hole, W, H); +console.log(`already-grained patch, added ${addedAgain.toFixed(2)}`); +assert.ok(addedAgain < 1.5, `a grained patch is left alone (added ${addedAgain.toFixed(2)})`); + +// 8. The fourth routing rule, the one only the engine knows: a mask no translation of the photo +// can clear. There is no patch to copy, so the mask is the model's whatever its area — and +// what the fast engine lays down instead has to be the smooth field of its own rim, with no +// direction in it. That is what the fan (radial rays converging on the centre) and the +// repeated stamp of a walked translation both are, and neither may come back from here. +const wide = frame(flat, { r: 250 }); // wider than half the crop in every direction: no shift clears it +// The mark the brush left behind, so "rebuilt" can be told from "left standing": everything under +// the brush is painted dark before the fill is asked for. +const marked = frame(flat, { r: 250 }); +for (let i = 0; i < W * H; i++) { + if (!marked.hole[i]) continue; + const p = i * 4; + marked.rgba[p] = marked.rgba[p + 1] = marked.rgba[p + 2] = 30; +} +const wideFill = fastLocalInpaint(marked.rgba, W, H, marked.hole); +assert.equal(localFillWasFlat(), true, 'a hole that no translation clears must report no donor'); +assert.equal(localFillWasFlat(), true, 'the flag has to survive the call the router reads it after'); +let leftStanding = 0, farFromRim = 0; +for (let i = 0; i < W * H; i++) { + if (!marked.hole[i]) continue; + if (wideFill[i * 4] <= 31) leftStanding++; + if (Math.abs(wideFill[i * 4] - 128) > 60) farFromRim++; +} +assert.equal(leftStanding, 0, `${leftStanding} pixels under the brush still carry the mark`); +assert.equal(farFromRim, 0, `${farFromRim} filled pixels land away from the ground's own tone`); + +// ...and the fill itself, on a ground with structure in it. What is asked of it is not accuracy +// — a hole this size is the model's, and SOR on a 500px disc converges short of the rim's own +// gradient, which is a smooth flattening and not an artifact — but that it invents no structure: +// it stays monotone across a ramp (no ridges, no crumpled planes) and it carries no fan. +const ramp = frame((x) => 30 + x * 0.4, { r: 250 }); +const rampFill = fastLocalInpaint(ramp.rgba, W, H, ramp.hole); +let worstRamp = 0, drops = 0, prev = null; +for (let y = 0; y < H; y++) { + for (let x = 0; x < W; x++) { + if (!ramp.hole[y * W + x]) continue; + const got = rampFill[(y * W + x) * 4]; + worstRamp = Math.max(worstRamp, Math.abs(got - luma(30 + x * 0.4))); + if (prev !== null && got < prev - 1) drops++; + prev = got; + } + prev = null; +} +console.log(`wide ramp fill: worst ${worstRamp.toFixed(1)} levels off, ${drops} non-monotone pixels`); +assert.equal(drops, 0, `the flat fill must not ripple across a ramp (${drops} pixels go backwards)`); +assert.ok(worstRamp < 40, `the residual of an unconverged wide solve stays a flattening (${worstRamp.toFixed(1)} levels)`); + +const striped = frame((x, y) => 128 + Math.sin(y * 0.6) * 24, { r: 250 }); +const stripedFill = fastLocalInpaint(striped.rgba, W, H, striped.hole); +const bins = new Array(72).fill(0), cnt = new Array(72).fill(0); +for (let y = 0; y < H; y++) { + for (let x = 0; x < W; x++) { + const i = y * W + x; + if (!striped.hole[i]) continue; + const dx = x - 256, dy = y - 256; + const d = Math.hypot(dx, dy) / 250; + if (d < 0.25 || d > 0.8) continue; + const a = Math.round(((Math.atan2(dy, dx) + Math.PI) / (2 * Math.PI)) * 72) % 72; + bins[a] += stripedFill[i * 4]; + cnt[a]++; + } +} +const prof = bins.filter((_, i) => cnt[i]).map((v, i) => v / cnt[i]); +const pm = prof.reduce((a, b) => a + b, 0) / prof.length; +const fan = (100 * Math.sqrt(prof.reduce((a, b) => a + (b - pm) ** 2, 0) / prof.length)) / pm; +console.log(`wide striped fill: angular spread ${fan.toFixed(1)}% of the mean (the fan measure)`); +assert.ok(fan < 3, `a wide hole comes back without fan sectors (${fan.toFixed(1)}% — the old fill ran 9-39%)`); + +console.log('PASS heal-route-check'); diff --git a/docker/frontend/scripts/heal-texture-check.mjs b/docker/frontend/scripts/heal-texture-check.mjs new file mode 100644 index 0000000..6130ef9 --- /dev/null +++ b/docker/frontend/scripts/heal-texture-check.mjs @@ -0,0 +1,91 @@ +// The hole sits in the busy half of the frame, and other places to take a donor from are a short +// hop away in the calm half, at the same tone. What the repair has to take from the photo is the +// texture it will sit among, so three things must hold: every pixel under the brush is rebuilt, +// the repair carries the surrounding texture rather than a slab laid over it (a donor picked on +// brightness and gradient alone lands a smooth dome in the middle of a ripple — the patch looks +// right against the rim it touches and wrong against everything else), and the tone the repair +// leaves behind is the photo's own, not the mark's. +// +// node scripts/heal-texture-check.mjs +// INPAINT_SRC=/tmp/old-inpaint.ts node scripts/heal-texture-check.mjs (must fail on the old engine) +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import ts from 'typescript'; + +const SRC = new URL(process.env.INPAINT_SRC ?? '../src/engine/inpaint.ts', import.meta.url); +const dir = mkdtempSync(join(tmpdir(), 'heal-texture-')); +writeFileSync( + join(dir, 'inpaint.mjs'), + ts.transpileModule(readFileSync(SRC, 'utf8'), { + compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2022 }, + }).outputText +); +const { fastLocalInpaint } = await import(pathToFileURL(join(dir, 'inpaint.mjs')).href); + +const W = 256, H = 192, CX = 170, CY = 96, R = 32; +const BUSY_FROM = 96; // everything left of this column is calm, and it is only a hop away +const truth = new Uint8Array(W * H * 4); +const dirty = new Uint8Array(W * H * 4); +const hole = new Uint8Array(W * H); +const tone = (x, y) => 120 + 30 * Math.sin((x + y) * 0.05); +for (let y = 0; y < H; y++) { + for (let x = 0; x < W; x++) { + const i = y * W + x; + const busy = x >= BUSY_FROM + ? 22 * Math.sin(x * 1.3 + y * 1.7) + 12 * Math.sin(x * 2.1 - y * 0.9) + : 0; + const v = Math.min(255, Math.max(0, tone(x, y) + busy)); + for (const px of [truth, dirty]) { + px[i * 4] = px[i * 4 + 1] = px[i * 4 + 2] = v; + px[i * 4 + 3] = 255; + } + // the mark: a red the grey photo cannot produce, so a pixel still wearing it was never rebuilt + if ((x - CX) ** 2 + (y - CY) ** 2 <= R * R) { + hole[i] = 1; + dirty[i * 4] = 200; + dirty[i * 4 + 1] = 20; + dirty[i * 4 + 2] = 20; + } + } +} + +const luma = (px, i) => 0.299 * px[i * 4] + 0.587 * px[i * 4 + 1] + 0.114 * px[i * 4 + 2]; +// High-pass energy: the grain and small structure, with the slow shading a patch match is +// supposed to ignore taken out. +const spread = (px, from, to) => { + let s = 0, ss = 0, n = 0; + for (let y = 1; y < H - 1; y++) { + for (let x = 1; x < W - 1; x++) { + const d = Math.hypot(x - CX, y - CY); + if (d < from || d > to) continue; + const i = y * W + x; + const hp = luma(px, i) - (luma(px, i - 1) + luma(px, i + 1) + luma(px, i - W) + luma(px, i + W)) / 4; + s += hp; ss += hp * hp; n++; + } + } + return Math.sqrt(Math.max(0, ss / n - (s / n) ** 2)); +}; + +const out = fastLocalInpaint(dirty, W, H, hole); + +let untouched = 0, sum = 0, n = 0, truthSum = 0; +for (let i = 0; i < W * H; i++) { + if (!hole[i]) continue; + if (out[i * 4] === 200 && out[i * 4 + 1] === 20) untouched++; + sum += luma(out, i); + truthSum += luma(truth, i); + n++; +} + +const inner = spread(out, 0, R * 0.8), ring = spread(out, R * 1.3, R * 2.2); +const ratio = inner / ring; +const toneMiss = Math.abs(sum / n - truthSum / n); +console.log(`{ untouched: ${untouched}, texture: ${ratio.toFixed(2)}, inner: ${inner.toFixed(2)}, ring: ${ring.toFixed(2)}, toneMiss: ${toneMiss.toFixed(2)} }`); + +assert.equal(untouched, 0, 'every pixel under the brush must be rebuilt'); +assert.ok(ratio > 0.6, `the repair must carry the surrounding texture, not a dome (texture ${ratio.toFixed(2)})`); +assert.ok(toneMiss < 3, `the repair must land on the photo's own tone (off by ${toneMiss.toFixed(2)})`); +console.log('heal texture match: ok'); diff --git a/docker/frontend/src/App.tsx b/docker/frontend/src/App.tsx index dab8c46..1e548a6 100644 --- a/docker/frontend/src/App.tsx +++ b/docker/frontend/src/App.tsx @@ -519,7 +519,7 @@ export function Workspace() { const [brushTool, setBrushTool] = useState<'heal' | 'mosaic' | null>(null); const [healR, setHealR] = useState(HEAL_DEFAULT_R); const [mosaicR, setMosaicR] = useState(MOSAIC_DEFAULT_R); - const [aiInpaint, setAiInpaint] = useState(() => recall('recipescam.heal.ai') === 'true'); + const [aiInpaint, setAiInpaint] = useState(() => recall('recipescam.heal.ai') !== 'false'); useEffect(() => { setAiInpaintMode(aiInpaint); keep('recipescam.heal.ai', String(aiInpaint)); diff --git a/docker/frontend/src/engine/exportEngine.ts b/docker/frontend/src/engine/exportEngine.ts index ecbc252..fd3d690 100644 --- a/docker/frontend/src/engine/exportEngine.ts +++ b/docker/frontend/src/engine/exportEngine.ts @@ -1079,13 +1079,20 @@ async function applyHealToBase( const cSpanW = mx1 - mx0; const cSpanH = my1 - my0; const cSpan = Math.max(cSpanW, cSpanH); - const margin = Math.max(48, Math.min(192, Math.round(Math.max(cWidest * 2.5, cSpan * 0.4) + 24))); - const srcLeft = Math.max(boxLeft, Math.floor(mx0 - margin)); - const srcTop = Math.max(boxTop, Math.floor(my0 - margin)); - const srcRight = Math.min(boxRight, Math.ceil(mx1 + margin)); - const srcBottom = Math.min(boxBottom, Math.ceil(my1 + margin)); - const srcW = srcRight - srcLeft; - const srcH = srcBottom - srcTop; + // Provide 60%–80% context padding (min 64px) around the cluster with dimensions divisible by 8 + // so both the Fast engine and LaMa ONNX have ample surrounding context without boundary distortion. + const padX = Math.max(64, Math.round(cSpanW * 0.75)); + const padY = Math.max(64, Math.round(cSpanH * 0.75)); + let srcLeft = Math.max(boxLeft, Math.floor(mx0 - padX)); + let srcTop = Math.max(boxTop, Math.floor(my0 - padY)); + let srcRight = Math.min(boxRight, Math.ceil(mx1 + padX)); + let srcBottom = Math.min(boxBottom, Math.ceil(my1 + padY)); + let srcW = srcRight - srcLeft; + let srcH = srcBottom - srcTop; + srcW = Math.max(8, srcW - (srcW % 8)); + srcH = Math.max(8, srcH - (srcH % 8)); + if (srcLeft + srcW > boxRight) srcLeft = Math.max(boxLeft, boxRight - srcW); + if (srcTop + srcH > boxBottom) srcTop = Math.max(boxTop, boxBottom - srcH); if (srcW >= 8 && srcH >= 8) { const { w: mw, h: mh } = inpaintCropSize(srcW, srcH); @@ -1128,7 +1135,11 @@ async function applyHealToBase( } const patchKey = `${mw}x${mh}_` + cluster.map((s) => `${s.x.toFixed(4)},${s.y.toFixed(4)},${s.r.toFixed(4)}`).join(';'); (hole as { patchKey?: string }).patchKey = patchKey; - const patch = await inpaintCrop(cropPixels, mw, mh, hole).catch(() => null); + // What the engine router is told: how much of the frame is under this cluster's + // brush. A crop is a tight window around its hole, so its own area says nothing + // about whether the mark is a speck or an object (see engine/inpaint.ts). + const brushArea = cluster.reduce((sum, s) => sum + Math.PI * (s.r * width) ** 2, 0); + const patch = await inpaintCrop(cropPixels, mw, mh, hole, brushArea / (width * height)).catch(() => null); if (patch) { const patchSurface = tempOwn(createSurface(mw, mh)); if (patchSurface && patchSurface.getCanvas().writePixels(patch, mw, mh, 0, 0)) { diff --git a/docker/frontend/src/engine/inpaint.ts b/docker/frontend/src/engine/inpaint.ts index 5db0f3d..7276299 100644 --- a/docker/frontend/src/engine/inpaint.ts +++ b/docker/frontend/src/engine/inpaint.ts @@ -1,6 +1,6 @@ -// FX > HEAL: Proximity Harmonic Dirichlet Solve + Border-to-Center Inward Detail & Ripple Propagation. -// Replaces heavy 28MB external neural weights with pure in-engine TypeScript execution. -// Runs in ~5-10ms with <2MB RAM footprint. 100% crash-free, zero iOS WebKit reload, zero lag. +// FX > HEAL: exemplar patch transfer taken from the photo's own clean pixels — donor search, +// and a directional walk + harmonic seam solve when the hole is wider than any clean patch. +// Pure in-engine TypeScript, no external neural weights: a few ms on a spot, <2MB of buffers. export const INPAINT_MAX_SIDE = 512; export const INPAINT_REF_LONG = 2048; @@ -25,198 +25,55 @@ export function onClearInpaint(cb: ClearCallback): void { } /** - * Applies coherent high-frequency micro-contrast and photographic sensor grain - * from the nearest clean boundary donors onto a base inpaint field. - * Uses 2-pass Euclidean Vector Distance Transform (EDT) and 5-tap separable Gaussian high-pass filter. + * Does the hole land entirely on clean pixels once the photo is shifted by (dx, dy)? Walked at + * `step`: the sweep the search runs on every candidate is coarse, and only the candidates that + * win on cost are re-walked at 1. A mask that covers half the crop leaves no room for a + * translation that clears it, and this is where that is found out. */ -export function applyCoherentTexture( - baseRgba: Uint8Array, - origRgba: Uint8Array | Uint8ClampedArray, +function patchCleanAt( + hole: Uint8Array, + clean: Uint8Array, w: number, h: number, - hole: Uint8Array -): Uint8Array { - const total = w * h; - const out = new Uint8Array(4 * total); - out.set(baseRgba); - - const clean = new Uint8Array(total); - let holeCount = 0; - for (let i = 0; i < total; i++) { - if (!hole[i]) clean[i] = 1; - else holeCount++; - } - if (holeCount === 0) return out; - - // 1. Two-pass Euclidean Vector Distance Transform (EDT) - const ny = new Int16Array(total); - const nx = new Int16Array(total); - const dsq = new Int32Array(total); - - for (let y = 0; y < h; y++) { + xmin: number, + ymin: number, + xmax: number, + ymax: number, + dx: number, + dy: number, + step: number +): boolean { + for (let y = ymin; y <= ymax; y += step) { + const sy = y + dy; + if (sy < 0 || sy >= h) { + const row = y * w; + for (let x = xmin; x <= xmax; x += step) if (hole[row + x]) return false; + continue; + } const row = y * w; - for (let x = 0; x < w; x++) { - const idx = row + x; - if (clean[idx]) { - ny[idx] = y; - nx[idx] = x; - dsq[idx] = 0; - } else { - dsq[idx] = 1000000000; - } + const sRow = sy * w; + for (let x = xmin; x <= xmax; x += step) { + if (!hole[row + x]) continue; + const sx = x + dx; + if (sx < 0 || sx >= w || !clean[sRow + sx]) return false; } } - - // Forward sweep (top-left to bottom-right) - for (let y = 0; y < h; y++) { - const row = y * w; - for (let x = 0; x < w; x++) { - const idx = row + x; - let curD = dsq[idx]; - let curNy = ny[idx]; - let curNx = nx[idx]; - - if (y > 0) { - const nIdx = idx - w; - const c = (y - ny[nIdx]) * (y - ny[nIdx]) + (x - nx[nIdx]) * (x - nx[nIdx]); - if (c < curD) { curD = c; curNy = ny[nIdx]; curNx = nx[nIdx]; } - } - if (x > 0) { - const nIdx = idx - 1; - const c = (y - ny[nIdx]) * (y - ny[nIdx]) + (x - nx[nIdx]) * (x - nx[nIdx]); - if (c < curD) { curD = c; curNy = ny[nIdx]; curNx = nx[nIdx]; } - } - if (y > 0 && x > 0) { - const nIdx = idx - w - 1; - const c = (y - ny[nIdx]) * (y - ny[nIdx]) + (x - nx[nIdx]) * (x - nx[nIdx]); - if (c < curD) { curD = c; curNy = ny[nIdx]; curNx = nx[nIdx]; } - } - if (y > 0 && x < w - 1) { - const nIdx = idx - w + 1; - const c = (y - ny[nIdx]) * (y - ny[nIdx]) + (x - nx[nIdx]) * (x - nx[nIdx]); - if (c < curD) { curD = c; curNy = ny[nIdx]; curNx = nx[nIdx]; } - } - - dsq[idx] = curD; - ny[idx] = curNy; - nx[idx] = curNx; - } - } - - // Backward sweep (bottom-right to top-left) - for (let y = h - 1; y >= 0; y--) { - const row = y * w; - for (let x = w - 1; x >= 0; x--) { - const idx = row + x; - let curD = dsq[idx]; - let curNy = ny[idx]; - let curNx = nx[idx]; - - if (y < h - 1) { - const nIdx = idx + w; - const c = (y - ny[nIdx]) * (y - ny[nIdx]) + (x - nx[nIdx]) * (x - nx[nIdx]); - if (c < curD) { curD = c; curNy = ny[nIdx]; curNx = nx[nIdx]; } - } - if (x < w - 1) { - const nIdx = idx + 1; - const c = (y - ny[nIdx]) * (y - ny[nIdx]) + (x - nx[nIdx]) * (x - nx[nIdx]); - if (c < curD) { curD = c; curNy = ny[nIdx]; curNx = nx[nIdx]; } - } - if (y < h - 1 && x < w - 1) { - const nIdx = idx + w + 1; - const c = (y - ny[nIdx]) * (y - ny[nIdx]) + (x - nx[nIdx]) * (x - nx[nIdx]); - if (c < curD) { curD = c; curNy = ny[nIdx]; curNx = nx[nIdx]; } - } - if (y < h - 1 && x > 0) { - const nIdx = idx + w - 1; - const c = (y - ny[nIdx]) * (y - ny[nIdx]) + (x - nx[nIdx]) * (x - nx[nIdx]); - if (c < curD) { curD = c; curNy = ny[nIdx]; curNx = nx[nIdx]; } - } - - dsq[idx] = curD; - ny[idx] = curNy; - nx[idx] = curNx; - } - } - - // 2. High-pass filter on origRgba: 5-tap separable Gaussian filter [1, 4, 6, 4, 1] / 16 - const tempR = new Float32Array(total); - const tempG = new Float32Array(total); - const tempB = new Float32Array(total); - - // Horizontal blur pass - for (let y = 0; y < h; y++) { - const row = y * w; - for (let x = 0; x < w; x++) { - const idx = row + x; - const xm2 = (row + Math.max(0, x - 2)) << 2; - const xm1 = (row + Math.max(0, x - 1)) << 2; - const x0 = idx << 2; - const xp1 = (row + Math.min(w - 1, x + 1)) << 2; - const xp2 = (row + Math.min(w - 1, x + 2)) << 2; - - tempR[idx] = (origRgba[xm2] + origRgba[xm1] * 4 + origRgba[x0] * 6 + origRgba[xp1] * 4 + origRgba[xp2]) * 0.0625; - tempG[idx] = (origRgba[xm2 + 1] + origRgba[xm1 + 1] * 4 + origRgba[x0 + 1] * 6 + origRgba[xp1 + 1] * 4 + origRgba[xp2 + 1]) * 0.0625; - tempB[idx] = (origRgba[xm2 + 2] + origRgba[xm1 + 2] * 4 + origRgba[x0 + 2] * 6 + origRgba[xp1 + 2] * 4 + origRgba[xp2 + 2]) * 0.0625; - } - } - - const meanR = new Float32Array(total); - const meanG = new Float32Array(total); - const meanB = new Float32Array(total); - - // Vertical blur pass - for (let y = 0; y < h; y++) { - const row = y * w; - const ym2 = Math.max(0, y - 2) * w; - const ym1 = Math.max(0, y - 1) * w; - const yp1 = Math.min(h - 1, y + 1) * w; - const yp2 = Math.min(h - 1, y + 2) * w; - - for (let x = 0; x < w; x++) { - const idx = row + x; - meanR[idx] = (tempR[ym2 + x] + tempR[ym1 + x] * 4 + tempR[idx] * 6 + tempR[yp1 + x] * 4 + tempR[yp2 + x]) * 0.0625; - meanG[idx] = (tempG[ym2 + x] + tempG[ym1 + x] * 4 + tempG[idx] * 6 + tempG[yp1 + x] * 4 + tempG[yp2 + x]) * 0.0625; - meanB[idx] = (tempB[ym2 + x] + tempB[ym1 + x] * 4 + tempB[idx] * 6 + tempB[yp1 + x] * 4 + tempB[yp2 + x]) * 0.0625; - } - } - - // 3. Superimpose high-pass texture onto baseRgba - for (let i = 0; i < total; i++) { - if (!clean[i]) { - const donor = ny[i] * w + nx[i]; - const dp = donor << 2; - - const texR = origRgba[dp] - meanR[donor]; - const texG = origRgba[dp + 1] - meanG[donor]; - const texB = origRgba[dp + 2] - meanB[donor]; - - const p = i << 2; - const r = Math.round(baseRgba[p] + texR); - const g = Math.round(baseRgba[p + 1] + texG); - const b = Math.round(baseRgba[p + 2] + texB); - - out[p] = r < 0 ? 0 : r > 255 ? 255 : r; - out[p + 1] = g < 0 ? 0 : g > 255 ? 255 : g; - out[p + 2] = b < 0 ? 0 : b > 255 ? 255 : b; - out[p + 3] = 255; - } - } - - return out; + return true; } /** * FAST ENGINE: True Exemplar Patch Transfer + Seamless Lighting Match. * Industry benchmark for spot healing (Adobe Photoshop / Lightroom standard): - * 1. Scans concentric rings across 32 directions around the hole to find the optimal clean - * source patch with matching texture, grain, and gradient alignment (0% hole overlap). - * 2. Transfers 100% genuine photographic pixels from the source patch. - * 3. Seamless Lighting Harmonization: Interpolates boundary difference field via Inverse Distance - * Weighting (IDW) to smoothly match lighting, exposure, and color across the hole. - * 4. Yields authentic, razor-sharp photographic texture with zero crumpled paper facets, - * zero comb lines, and zero synthetic blur. - * Runs in pure TypeScript in ~3-10ms with <2MB memory. + * 1. Scans concentric rings across 32 directions around the hole for the nearest fully clean + * source patch whose grain and small structure match the halo the repair will blend into, not + * just the pixels it touches (0% hole overlap). A donor that is merely the right brightness + * but the wrong texture is what lands a smooth dome in the middle of a ripple. + * 2. Transfers 100% genuine photographic pixels from that patch. + * 3. Harmonizes the seam as a seamless clone: the rim's own mismatch is solved over the hole, so + * lighting, exposure and color meet the photo exactly where the patch touches it. + * 4. When no patch is fully clean — a hole wider than its surroundings — it hands over to the + * directional walk + harmonic seam solve below, instead of averaging the rim. + * Pure TypeScript with no external weights: ~3-10ms on a spot, <2MB of buffers. */ export function fastLocalInpaint( rgba: Uint8Array | Uint8ClampedArray, @@ -287,6 +144,58 @@ export function fastLocalInpaint( } // 2. Candidate Source Patch Search (Expanding concentric rings, 32 angles) + // + // What the patch has to match is the texture it will sit among, not only the few pixels it + // touches: a donor cut from a calm patch of the same water lands on the rim's own average and + // reads as a smooth dome in the middle of a ripple. So the cost is read on a halo wider than + // the lip, and what is compared on it is the local high-pass — grain and small structure with + // the shading taken out — how strong it is and how close its phase runs. + const lum = new Float32Array(total); + for (let i = 0; i < total; i++) { + const p = i << 2; + lum[i] = 0.299 * rgba[p] + 0.587 * rgba[p + 1] + 0.114 * rgba[p + 2]; + } + const hp = new Float32Array(total); // luma above its own 3x3 mean; garbage where hpOk is 0 + const hpOk = new Uint8Array(total); + for (let y = 1; y < h - 1; y++) { + for (let x = 1; x < w - 1; x++) { + const i = y * w + x; + if (!clean[i]) continue; + let s = 0, ok = true; + for (let t = -1; t <= 1 && ok; t++) { + const row = i + t * w; + for (let u = -1; u <= 1; u++) { + const q = row + u; + if (!clean[q]) { ok = false; break; } + s += lum[q]; + } + } + if (!ok) continue; + hp[i] = lum[i] - s / 9; + hpOk[i] = 1; + } + } + + // The halo the repair blends into, sampled on a grid — dense enough for a stable spread, coarse + // enough to keep the search under a few ms. + const ringR = Math.max(3, Math.min(8, Math.round(maxSpan / 6))); + const mX0 = Math.max(0, xmin - ringR), mX1 = Math.min(w - 1, xmax + ringR); + const mY0 = Math.max(0, ymin - ringR), mY1 = Math.min(h - 1, ymax + ringR); + const meas: number[] = []; + let sumQ = 0, sumQQ = 0; + for (let y = mY0; y <= mY1; y += 2) { + const row = y * w; + for (let x = mX0; x <= mX1; x += 2) { + const idx = row + x; + if (!hpOk[idx]) continue; + meas.push(idx); + sumQ += hp[idx]; + sumQQ += hp[idx] * hp[idx]; + } + } + const nQ = meas.length; + const stdQ = nQ ? Math.sqrt(Math.max(0, sumQQ / nQ - (sumQ / nQ) ** 2)) : 0; + let bestDx = 0, bestDy = 0; let minCost = Infinity; let found = false; @@ -302,57 +211,51 @@ export function fastLocalInpaint( const dy = Math.round(Math.sin(angle) * d); // Verify source patch is 100% clean and in bounds - let valid = true; - for (let y = ymin; y <= ymax && valid; y++) { + if (!patchCleanAt(hole, clean, w, h, xmin, ymin, xmax, ymax, dx, dy, 3)) continue; + + // Evaluate the candidate on the halo the repair blends into: the spread of the donor's + // high-pass against the halo's (the same texture, not a calmer one — that is the dome), + // then the phase it runs at, then the plain colour difference the seam solve partly pays + // for anyway. The distance term keeps the donor from being dragged across the photo for a + // fraction of a point, and the max-diff penalty keeps a foreign edge out of the hole. + let n = 0, nS = 0, sumDiff = 0, sumHp = 0, sumS = 0, sumSS = 0, maxDiff = 0; + for (let i = 0; i < nQ; i++) { + const idx = meas[i]; + const y = (idx / w) | 0; + const sx = idx - y * w + dx; const sy = y + dy; - if (sy < 0 || sy >= h) { - for (let x = xmin; x <= xmax; x++) { - if (hole[y * w + x]) { valid = false; break; } - } - continue; - } - const row = y * w; - const sRow = sy * w; - for (let x = xmin; x <= xmax; x++) { - if (!hole[row + x]) continue; - const sx = x + dx; - if (sx < 0 || sx >= w || !clean[sRow + sx]) { - valid = false; - break; - } - } - } - if (!valid) continue; - - // Evaluate boundary match cost - let sumDiff = 0; - let maxDiff = 0; - let count = 0; - for (let i = 0; i < band.length; i += 2) { - const idx = band[i]; - const bx = idx % w; - const by = (idx / w) | 0; - const sx = bx + dx; - const sy = by + dy; - if (sx < 0 || sx >= w || sy < 0 || sy >= h || !clean[sy * w + sx]) continue; - - const bp = idx << 2; - const sp = (sy * w + sx) << 2; - const dr = Math.abs(rgba[bp] - rgba[sp]); - const dg = Math.abs(rgba[bp + 1] - rgba[sp + 1]); - const db = Math.abs(rgba[bp + 2] - rgba[sp + 2]); - const diff = dr + dg + db; + if (sx < 0 || sx >= w || sy < 0 || sy >= h) continue; + const s = sy * w + sx; + if (!clean[s]) continue; + const bp = idx << 2, sp = s << 2; + const diff = + Math.abs(rgba[bp] - rgba[sp]) + + Math.abs(rgba[bp + 1] - rgba[sp + 1]) + + Math.abs(rgba[bp + 2] - rgba[sp + 2]); sumDiff += diff; if (diff > maxDiff) maxDiff = diff; - count++; + n++; + if (hpOk[s]) { + const dh = hp[idx] - hp[s]; + sumHp += dh < 0 ? -dh : dh; + sumS += hp[s]; + sumSS += hp[s] * hp[s]; + nS++; + } } - if (count > 0) { - const avgDiff = sumDiff / count; - // Avoid foreign high-contrast edges + if (n > 7) { + const stdS = nS ? Math.sqrt(Math.max(0, sumSS / nS - (sumS / nS) ** 2)) : 0; const penalty = maxDiff > 90 ? 400 : (maxDiff > 50 ? 80 : 0); - const cost = avgDiff + d * 0.08 + penalty; - if (cost < minCost) { + const cost = + Math.abs(stdQ - stdS) * 2.0 + + (nS ? sumHp / nS : 0) * 0.6 + + (sumDiff / n) * 0.3 + + d * 0.12 + + penalty; + // The candidate is only taken once it holds up at full resolution, so the sweep can + // throw out the thousands of shifts that never had a chance without walking them. + if (cost < minCost && patchCleanAt(hole, clean, w, h, xmin, ymin, xmax, ymax, dx, dy, 1)) { minCost = cost; bestDx = dx; bestDy = dy; @@ -365,164 +268,255 @@ export function fastLocalInpaint( const out = new Uint8Array(rgba.length); out.set(rgba); + // What the reader of this mask needs to know: whether the photo had a patch to give. It is + // the one signal the router cannot read off the pixels — a wide mask on calm ground passes + // every threshold and still leaves the fast engine with nothing to copy — so it is reported + // here and read in inpaintCrop. + lastFillFlat = !found; + if (found) { - // 3A. TRUE PATCH TRANSFER + SEAMLESS LIGHTING ADJUSTMENT (Photoshop / Lightroom Standard) - const borderSamples: { x: number; y: number; dr: number; dg: number; db: number }[] = []; - for (let i = 0; i < band.length; i++) { - const idx = band[i]; - const bx = idx % w; - const by = (idx / w) | 0; - const sx = bx + bestDx; - const sy = by + bestDy; - if (sx < 0 || sx >= w || sy < 0 || sy >= h) continue; - - const bp = idx << 2; - const sp = (sy * w + sx) << 2; - borderSamples.push({ - x: bx, - y: by, - dr: rgba[bp] - rgba[sp], - dg: rgba[bp + 1] - rgba[sp + 1], - db: rgba[bp + 2] - rgba[sp + 2], - }); - } - - const sampleStep = Math.max(1, Math.floor(borderSamples.length / 32)); - const activeSamples: { x: number; y: number; dr: number; dg: number; db: number }[] = []; - for (let i = 0; i < borderSamples.length; i += sampleStep) { - activeSamples.push(borderSamples[i]); - } - + // 3A. PATCH TRANSFER, SEAMED. The patch is the photo's own detail; the one thing it + // cannot carry is where it meets the photo. What meets the photo there is solved rather + // than weighted: out = patch + φ, φ harmonic over the hole with φ = photo − patch on the + // rim — the seamless clone, the same solve the fallback below uses. The inverse-distance + // average this replaced spread the rim's *mean* over the hole: a smooth field with no + // structure in it, which is why the repair could sit on the photo's tone and still show + // every edge inside it stopping dead at the rim. + const donor = new Uint8Array(rgba.length); + donor.set(rgba); + const hasDonor = new Uint8Array(total); + const holeIdx: number[] = []; for (let y = ymin; y <= ymax; y++) { const row = y * w; - const sRow = (y + bestDy) * w; + const sRow = (y + bestDy) * w + bestDx; for (let x = xmin; x <= xmax; x++) { const idx = row + x; if (!hole[idx]) continue; - - let sumDr = 0, sumDg = 0, sumDb = 0, sumW = 0; - for (let i = 0; i < activeSamples.length; i++) { - const s = activeSamples[i]; - const distSq = (x - s.x) * (x - s.x) + (y - s.y) * (y - s.y); - const weight = 1.0 / (distSq + 1.0); - sumDr += s.dr * weight; - sumDg += s.dg * weight; - sumDb += s.db * weight; - sumW += weight; - } - - const sIdx = sRow + (x + bestDx); - const sp = sIdx << 2; const p = idx << 2; - - const offsetR = sumW > 0 ? sumDr / sumW : 0; - const offsetG = sumW > 0 ? sumDg / sumW : 0; - const offsetB = sumW > 0 ? sumDb / sumW : 0; - - const r = Math.round(rgba[sp] + offsetR); - const g = Math.round(rgba[sp + 1] + offsetG); - const b = Math.round(rgba[sp + 2] + offsetB); - - out[p] = r < 0 ? 0 : r > 255 ? 255 : r; - out[p + 1] = g < 0 ? 0 : g > 255 ? 255 : g; - out[p + 2] = b < 0 ? 0 : b > 255 ? 255 : b; - out[p + 3] = 255; + const sp = (sRow + x) << 2; + donor[p] = rgba[sp]; + donor[p + 1] = rgba[sp + 1]; + donor[p + 2] = rgba[sp + 2]; + donor[p + 3] = 255; + hasDonor[idx] = 1; + holeIdx.push(idx); } } + + seamlessClone(rgba, out, donor, hasDonor, hole, w, h, holeIdx, band, Math.max(holeW, holeH), (q) => { + const y = (q / w) | 0; + const sx = q - y * w + bestDx; + const sy = y + bestDy; + if (sx < 0 || sx >= w || sy < 0 || sy >= h) return -1; + const s = sy * w + sx; + return clean[s] ? s : -1; + }); } else { - // 3B. SMOOTH HARMONIC COONS INTERPOLATION FALLBACK (Zero facets, zero crumpled paper) - const baseR = new Float32Array(total); - const baseG = new Float32Array(total); - const baseB = new Float32Array(total); - - for (let i = 0; i < total; i++) { - if (clean[i]) { - const p = i << 2; - baseR[i] = rgba[p]; - baseG[i] = rgba[p + 1]; - baseB[i] = rgba[p + 2]; - } - } - - for (let y = ymin; y <= ymax; y++) { - const row = y * w; - for (let x = xmin; x <= xmax; x++) { - const idx = row + x; - if (clean[idx]) continue; - - let xl = x, xr = x, yt = y, yb = y; - while (xl >= 0 && !clean[row + xl]) xl--; - while (xr < w && !clean[row + xr]) xr++; - while (yt >= 0 && !clean[yt * w + x]) yt--; - while (yb < h && !clean[yb * w + x]) yb++; - - let sumR = 0, sumG = 0, sumB = 0, sumW = 0; - if (xl >= 0) { const d = x - xl; const wt = 1.0 / (d * d); const p = (row + xl) << 2; sumR += rgba[p] * wt; sumG += rgba[p + 1] * wt; sumB += rgba[p + 2] * wt; sumW += wt; } - if (xr < w) { const d = xr - x; const wt = 1.0 / (d * d); const p = (row + xr) << 2; sumR += rgba[p] * wt; sumG += rgba[p + 1] * wt; sumB += rgba[p + 2] * wt; sumW += wt; } - if (yt >= 0) { const d = y - yt; const wt = 1.0 / (d * d); const p = (yt * w + x) << 2; sumR += rgba[p] * wt; sumG += rgba[p + 1] * wt; sumB += rgba[p + 2] * wt; sumW += wt; } - if (yb < h) { const d = yb - y; const wt = 1.0 / (d * d); const p = (yb * w + x) << 2; sumR += rgba[p] * wt; sumG += rgba[p + 1] * wt; sumB += rgba[p + 2] * wt; sumW += wt; } - - if (sumW > 0) { - baseR[idx] = sumR / sumW; - baseG[idx] = sumG / sumW; - baseB[idx] = sumB / sumW; - } - } - } - - const omega = 1.75; - const oneMinusOmega = 1.0 - omega; - for (let it = 0; it < 25; it++) { - for (let y = ymin; y <= ymax; y++) { - const row = y * w; - for (let x = xmin; x <= xmax; x++) { - const idx = row + x; - if (clean[idx]) continue; - - const rR = x < w - 1 ? baseR[idx + 1] : baseR[idx - 1]; - const rL = x > 0 ? baseR[idx - 1] : baseR[idx + 1]; - const rD = y < h - 1 ? baseR[idx + w] : baseR[idx - w]; - const rU = y > 0 ? baseR[idx - w] : baseR[idx + w]; - - baseR[idx] = oneMinusOmega * baseR[idx] + omega * (rR + rL + rD + rU) * 0.25; - - const gR = x < w - 1 ? baseG[idx + 1] : baseG[idx - 1]; - const gL = x > 0 ? baseG[idx - 1] : baseG[idx + 1]; - const gD = y < h - 1 ? baseG[idx + w] : baseG[idx - w]; - const gU = y > 0 ? baseG[idx - w] : baseG[idx + w]; - - baseG[idx] = oneMinusOmega * baseG[idx] + omega * (gR + gL + gD + gU) * 0.25; - - const bR = x < w - 1 ? baseB[idx + 1] : baseB[idx - 1]; - const bL = x > 0 ? baseB[idx - 1] : baseB[idx + 1]; - const bD = y < h - 1 ? baseB[idx + w] : baseB[idx - w]; - const bU = y > 0 ? baseB[idx - w] : baseB[idx + w]; - - baseB[idx] = oneMinusOmega * baseB[idx] + omega * (bR + bL + bD + bU) * 0.25; - } - } - } - - for (let y = ymin; y <= ymax; y++) { - const row = y * w; - for (let x = xmin; x <= xmax; x++) { - const idx = row + x; - if (!hole[idx]) continue; - const p = idx << 2; - const r = Math.round(baseR[idx]); - const g = Math.round(baseG[idx]); - const b = Math.round(baseB[idx]); - out[p] = r < 0 ? 0 : r > 255 ? 255 : r; - out[p + 1] = g < 0 ? 0 : g > 255 ? 255 : g; - out[p + 2] = b < 0 ? 0 : b > 255 ? 255 : b; - out[p + 3] = 255; - } - } + // 3B. NO DONOR — NOTHING TO COPY, SO NOTHING IS INVENTED. A hole wide enough that no + // translation clears it leaves the frame's own pixels as the only source, and every + // propagation tried here has failed the same way: a translation walked across the hole + // stamps the same texture down again once per shift, and inverse-distance or directional + // weighting leaves ridges and fan sectors running to the centre (the tangential spray, the + // "ripple" — measured, and reproduced on a textured frame at 3x the fan of the field + // below). What a hole this size can honestly be given is the smooth field its own rim + // already implies: ΔI = 0 inside with the rim held fixed, which carries no direction and so + // shows neither. Texture, and the edge a mask this wide usually straddles, are the model's + // job — the router hands this mask to the server whenever the fast engine had no donor. + // ponytail: flat by design, not by accident. Add a PatchMatch/LaMa fill at this call site + // only if the tier-1 offline path must keep texture on masks wider than half the crop. + laplaceFill(rgba, out, hole, w, h, xmin, ymin, xmax, ymax); } return out; } +/** + * ΔI = 0 inside the hole, I = the photo on the rim: the field a hole nobody can copy has to be + * given. SOR with the rim held fixed, seeded from the rim's own mean, no direction anywhere in + * it — which is exactly why it shows neither rays nor a repeated stamp. Seamless by + * construction: the rim it meets is the photo's own. + */ +function laplaceFill( + rgba: Uint8Array | Uint8ClampedArray, + out: Uint8Array, + hole: Uint8Array, + w: number, + h: number, + xmin: number, + ymin: number, + xmax: number, + ymax: number +): void { + const x0 = Math.max(0, xmin - 1), x1 = Math.min(w - 1, xmax + 1); + const y0 = Math.max(0, ymin - 1), y1 = Math.min(h - 1, ymax + 1); + const fr = new Float32Array(w * h); + const fg = new Float32Array(w * h); + const fb = new Float32Array(w * h); + let mr = 0, mg = 0, mb = 0, mn = 0; + for (let y = y0; y <= y1; y++) { + const row = y * w; + for (let x = x0; x <= x1; x++) { + const i = row + x; + if (hole[i]) continue; + const p = i << 2; + fr[i] = rgba[p]; + fg[i] = rgba[p + 1]; + fb[i] = rgba[p + 2]; + mr += rgba[p]; mg += rgba[p + 1]; mb += rgba[p + 2]; mn++; + } + } + if (mn) { + // Seeded from the rim's own mean: a hole wide enough to reach the frame leaves the rim + // hundreds of pixels away, and a cold start spends its sweeps walking out of black. + mr /= mn; mg /= mn; mb /= mn; + for (let y = ymin; y <= ymax; y++) { + const row = y * w; + for (let x = xmin; x <= xmax; x++) { + const i = row + x; + if (!hole[i]) continue; + fr[i] = mr; fg[i] = mg; fb[i] = mb; + } + } + } + const omega = 1.9, om = 1 - omega; // near-optimal SOR: the rim is a couple of hundred pixels away + // Sweeps scale with the depth of the hole, and stop well short of what SOR would need to + // converge one this wide. ponytail: the residual is a smooth flattening of the rim's own + // gradient, which is the ceiling this path accepts — a mask this big is the model's (the + // router sends it there), and what the local path owes is that it invents no structure. + const sweeps = Math.min(400, 32 + 2 * Math.max(xmax - xmin, ymax - ymin)); + for (let it = 0; it < sweeps; it++) { + for (let y = ymin; y <= ymax; y++) { + const row = y * w; + for (let x = xmin; x <= xmax; x++) { + const i = row + x; + if (!hole[i]) continue; + const ql = i - 1, qr = i + 1, qu = i - w, qd = i + w; + const ar = fr[ql] + fr[qr] + fr[qu] + fr[qd]; + const ag = fg[ql] + fg[qr] + fg[qu] + fg[qd]; + const ab = fb[ql] + fb[qr] + fb[qu] + fb[qd]; + fr[i] = om * fr[i] + omega * ar * 0.25; + fg[i] = om * fg[i] + omega * ag * 0.25; + fb[i] = om * fb[i] + omega * ab * 0.25; + } + } + } + for (let y = ymin; y <= ymax; y++) { + const row = y * w; + for (let x = xmin; x <= xmax; x++) { + const i = row + x; + if (!hole[i]) continue; + const p = i << 2; + const r = Math.round(fr[i]), g = Math.round(fg[i]), b = Math.round(fb[i]); + out[p] = r < 0 ? 0 : r > 255 ? 255 : r; + out[p + 1] = g < 0 ? 0 : g > 255 ? 255 : g; + out[p + 2] = b < 0 ? 0 : b > 255 ? 255 : b; + out[p + 3] = 255; + } + } +} + +/** + * The patch, seamed into the photo it was taken from — the seamless clone. `out = donor + φ` + * over the hole, where φ solves Laplace's equation inside the hole with φ = photo − donor + * held fixed on the rim: the rim's own mismatch, spread evenly over the hole, so the repair + * meets the photo exactly where they touch and the donor's detail is left standing inside it. + * `donor` carries the patch's pixels; every kind of donor answers the same call, whether the + * pixels came from one translation of the photo or from the walk's own passes. + */ +function seamlessClone( + rgba: Uint8Array | Uint8ClampedArray, + out: Uint8Array, + donor: Uint8Array, + hasDonor: Uint8Array, + hole: Uint8Array, + w: number, + h: number, + holeIdx: number[], + rim: number[], + span: number, + rimDonor: (q: number) => number +): void { + const rimSlot = new Int32Array(w * h).fill(-1); + const rimV = new Float32Array(3 * rim.length); + let mr = 0, mg = 0, mb = 0, rn = 0; + for (let i = 0; i < rim.length; i++) { + const q = rim[i]; + const s = rimDonor(q); + if (s < 0) continue; + const sp = s << 2, qp = q << 2; + const d0 = rgba[qp] - donor[sp]; + const d1 = rgba[qp + 1] - donor[sp + 1]; + const d2 = rgba[qp + 2] - donor[sp + 2]; + rimSlot[q] = i; + rimV[i * 3] = d0; + rimV[i * 3 + 1] = d1; + rimV[i * 3 + 2] = d2; + mr += d0; + mg += d1; + mb += d2; + rn++; + } + if (!rn) return; + + mr /= rn; + mg /= rn; + mb /= rn; + const phiR = new Float32Array(w * h); + const phiG = new Float32Array(w * h); + const phiB = new Float32Array(w * h); + for (let i = 0; i < holeIdx.length; i++) { + const idx = holeIdx[i]; + phiR[idx] = mr; + phiG[idx] = mg; + phiB[idx] = mb; + } + + // Gauss-Seidel with over-relaxation, the rim held fixed. The seed is only a start: what the + // old fill did was to keep it. + const sweeps = Math.min(160, 24 + span); + const omega = 1.9, om = 1 - omega; // near-optimal SOR: the rim is 300+ pixels away in a wide hole + for (let it = 0; it < sweeps; it++) { + for (let i = 0; i < holeIdx.length; i++) { + const idx = holeIdx[i]; + const x = idx - ((idx / w) | 0) * w; + const y = (idx / w) | 0; + let cnt = 0, ar = 0, ag = 0, ab = 0; + for (let t = 0; t < 4; t++) { + const nx = x + (t === 0 ? -1 : t === 1 ? 1 : 0); + const ny = y + (t === 2 ? -1 : t === 3 ? 1 : 0); + if (nx < 0 || nx >= w || ny < 0 || ny >= h) continue; + const q = ny * w + nx; + if (hole[q]) { + ar += phiR[q]; ag += phiG[q]; ab += phiB[q]; + cnt++; + } else { + const slot = rimSlot[q]; + if (slot < 0) continue; + ar += rimV[slot * 3]; ag += rimV[slot * 3 + 1]; ab += rimV[slot * 3 + 2]; + cnt++; + } + } + if (!cnt) continue; + phiR[idx] = om * phiR[idx] + (omega * ar) / cnt; + phiG[idx] = om * phiG[idx] + (omega * ag) / cnt; + phiB[idx] = om * phiB[idx] + (omega * ab) / cnt; + } + } + + for (let i = 0; i < holeIdx.length; i++) { + const idx = holeIdx[i]; + if (!hasDonor[idx]) continue; // no donor for this pixel: leave the photo's own there + const p = idx << 2; + const r = Math.round(donor[p] + phiR[idx]); + const g = Math.round(donor[p + 1] + phiG[idx]); + const b = Math.round(donor[p + 2] + phiB[idx]); + out[p] = r < 0 ? 0 : r > 255 ? 255 : r; + out[p + 1] = g < 0 ? 0 : g > 255 ? 255 : g; + out[p + 2] = b < 0 ? 0 : b > 255 ? 255 : b; + out[p + 3] = 255; + } +} + export function cropPlate( rgba: Uint8Array | Uint8ClampedArray, width: number, @@ -588,7 +582,7 @@ if (typeof addEventListener === 'function') { }); } -let isAiEnabled = false; +let isAiEnabled = true; export function setAiInpaintMode(enabled: boolean): void { if (isAiEnabled !== enabled) { @@ -601,6 +595,221 @@ export function getAiInpaintMode(): boolean { return isAiEnabled; } +// ---- Router: which of the two engines takes this mask ---- +// A dab of dust on open ground is the fast engine's job: it costs nothing, needs no server, and +// is already the better tool there. What the server model is for is a mask wide enough that the +// fast engine has no clean patch to copy, or one sitting across a real edge (a wire against sky, +// a branch against leaf) where the pixels have to be invented from the whole scene and not +// borrowed from a patch that is merely nearby. Three readings decide it, in the order the tier +// chart draws them: how much of the frame is under the brush, how much structure the rim carries +// against the window at large (an edge), and how busy the rim is outright (ground the engine can +// only ever copy a slab of). +export const AI_MAX_MASK_AREA = 0.035; // 3.5% of the frame, as the tier chart reads it +export const AI_EDGE_RATIO = 1.5; // rim carries structure this much above the window's own +export const AI_EDGE_FLOOR = 1.0; // ...and this much at all, or the ratio is noise talking +export const AI_TEXTURE_LIMIT = 6.0; // rim this busy on its own is ground the engine clones into + +function lumaAt(rgba: Uint8Array | Uint8ClampedArray, i: number): number { + const p = i << 2; + return 0.299 * rgba[p] + 0.587 * rgba[p + 1] + 0.114 * rgba[p + 2]; +} + +// Mean |luma - 3x3 luma| over the pixels that touch the mask (rim), and over the window at large. +// Their ratio is what says "this mask straddles an edge", and it is scale-free: a grainy photo +// raises both, so grain alone never sends a dab to the server. The rim is walked at full +// resolution — an edge is one pixel wide and a sampled grid steps straight over it. +function structureAt( + rgba: Uint8Array | Uint8ClampedArray, + w: number, + h: number, + hole: Uint8Array +): { rim: number; all: number } { + let xmin = w, ymin = h, xmax = -1, ymax = -1; + for (let y = 0; y < h; y++) { + const row = y * w; + for (let x = 0; x < w; x++) { + if (!hole[row + x]) continue; + if (x < xmin) xmin = x; + if (x > xmax) xmax = x; + if (y < ymin) ymin = y; + if (y > ymax) ymax = y; + } + } + if (xmax < 0) return { rim: 0, all: 0 }; + + const hpAt = (i: number): number => { + let m = 0; + for (let dy = -1; dy <= 1; dy++) { + for (let dx = -1; dx <= 1; dx++) m += lumaAt(rgba, i + dy * w + dx); + } + return Math.abs(lumaAt(rgba, i) - m / 9); + }; + + let rimSum = 0, rimN = 0; + for (let y = Math.max(1, ymin - 1); y <= Math.min(h - 2, ymax + 1); y++) { + const row = y * w; + for (let x = Math.max(1, xmin - 1); x <= Math.min(w - 2, xmax + 1); x++) { + const i = row + x; + if (hole[i]) continue; + if (hole[i - 1] || hole[i + 1] || hole[i - w] || hole[i + w]) { + rimSum += hpAt(i); + rimN++; + } + } + } + + let allSum = 0, allN = 0; + const step = Math.max(1, Math.floor(Math.min(w, h) / 96)); + for (let y = 1; y < h - 1; y += step) { + const row = y * w; + for (let x = 1; x < w - 1; x += step) { + const i = row + x; + if (hole[i]) continue; + allSum += hpAt(i); + allN++; + } + } + return { rim: rimN ? rimSum / rimN : 0, all: allN ? allSum / allN : 0 }; +} + +// Whether the last fastLocalInpaint had a donor to copy from. Read by the router below: a mask +// the fast engine could only lay its flat rim field under is the model's, whatever its area. +let lastFillFlat = false; + +/** The router's own reading of that flag, and the one the checks read. */ +export function localFillWasFlat(): boolean { + return lastFillFlat; +} + +export function aiTierNeeded( + rgba: Uint8Array | Uint8ClampedArray, + w: number, + h: number, + hole: Uint8Array, + holeFrac = 0 +): boolean { + let count = 0; + for (let i = 0; i < hole.length; i++) if (hole[i]) count++; + if (!count) return false; + // The area the tier chart means is the area of the *frame* under the brush; a crop is tight + // around its hole, so counting in crop pixels would put every dab in the model's column. Where + // the frame is unknown the crop is what we have, which reads as "big" and sends it to the model. + const frac = holeFrac > 0 ? holeFrac : count / (w * h); + if (frac > AI_MAX_MASK_AREA) return true; + const { rim, all } = structureAt(rgba, w, h, hole); + if (rim > AI_TEXTURE_LIMIT) return true; // busy ground: the engine lands a slab of somewhere else + return rim > AI_EDGE_FLOOR && rim > all * AI_EDGE_RATIO; +} + +// The brush leaves the object's own shadow behind, so the mask the model is given is grown a few +// pixels — a hair wider than the stroke, never wider than the feather that composites it back. +export function dilateMask( + mask: Uint8Array, + w: number, + h: number, + r: number +): Uint8Array { + if (r < 1) return mask; + const horiz = new Uint8Array(mask.length); + for (let y = 0; y < h; y++) { + const row = y * w; + for (let x = 0; x < w; x++) { + let on = 0; + const x0 = Math.max(0, x - r), x1 = Math.min(w - 1, x + r); + for (let k = x0; k <= x1 && !on; k++) if (mask[row + k]) on = 1; + horiz[row + x] = on; + } + } + const out = new Uint8Array(mask.length); + for (let y = 0; y < h; y++) { + const y0 = Math.max(0, y - r), y1 = Math.min(h - 1, y + r); + for (let x = 0; x < w; x++) { + let on = 0; + for (let k = y0; k <= y1 && !on; k++) if (horiz[k * w + x]) on = 255; + out[y * w + x] = on; + } + } + return out; +} + +// Grain. What the model returns is a perfectly clean surface, and a clean surface pasted into a +// photo whose every other pixel carries the sensor's own noise reads as a smudge however right +// the tone is. So the patch is given back the grain of the ground it was solved from — measured +// on the pixels around the mask, minus whatever the model left of its own. +export function matchGrain( + patch: Uint8Array, + src: Uint8Array, + mask: Uint8Array, + w: number, + h: number +): number { + const plane = w * h; + const sigma = (data: Uint8Array, at: (i: number) => boolean, stride: number): number => { + let sum = 0, sum2 = 0, n = 0; + for (let y = 1; y < h - 1; y += stride) { + const row = y * w; + for (let x = 1; x < w - 1; x += stride) { + const i = row + x; + if (!at(i)) continue; + const mean = + (data[i - w - 1] + data[i - w] + data[i - w + 1] + + data[i - 1] + data[i] + data[i + 1] + + data[i + w - 1] + data[i + w] + data[i + w + 1]) / 9; + const hp = data[i] - mean; + sum += hp; + sum2 += hp * hp; + n++; + } + } + if (n < 16) return 0; + const m = sum / n; + return Math.sqrt(Math.max(0, sum2 / n - m * m)); + }; + + const holeR = new Uint8Array(plane); + const holeG = new Uint8Array(plane); + const holeB = new Uint8Array(plane); + for (let i = 0; i < plane; i++) { + const s = i << 2; + holeR[i] = patch[s]; + holeG[i] = patch[s + 1]; + holeB[i] = patch[s + 2]; + } + const inHole = (i: number) => mask[i] !== 0; + const outside = (i: number) => + !mask[i] && + !!(mask[i - 1] || mask[i + 1] || mask[i - w] || mask[i + w] || + mask[i - w - 1] || mask[i - w + 1] || mask[i + w - 1] || mask[i + w + 1]); + + const rS = sigma(src, outside, 1); + const gS = sigma(src.subarray(plane), outside, 1); + const bS = sigma(src.subarray(2 * plane), outside, 1); + const rP = sigma(holeR, inHole, 2); + const gP = sigma(holeG, inHole, 2); + const bP = sigma(holeB, inHole, 2); + + const add = (s: number, p: number) => Math.min(16, Math.max(0, s - p)); + const aR = add(rS, rP), aG = add(gS, gP), aB = add(bS, bP); + if (aR + aG + aB < 0.05) return 0; + + let seed = 0x9e3779b9; + const rand = () => { + seed |= 0; + seed = (seed + 0x6d2b79f5) | 0; + let t = Math.imul(seed ^ (seed >>> 15), 1 | seed); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return (((t ^ (t >>> 14)) >>> 0) / 4294967296) * 2 - 1; + }; + for (let i = 0; i < plane; i++) { + if (!mask[i]) continue; + const s = i << 2; + patch[s] = Math.max(0, Math.min(255, Math.round(patch[s] + rand() * aR))); + patch[s + 1] = Math.max(0, Math.min(255, Math.round(patch[s + 1] + rand() * aG))); + patch[s + 2] = Math.max(0, Math.min(255, Math.round(patch[s + 2] + rand() * aB))); + } + return (aR + aG + aB) / 3; +} + function uint8ToBase64(bytes: Uint8Array): string { let binary = ''; const len = bytes.byteLength; @@ -626,14 +835,17 @@ export async function inpaintCrop( rgba: Uint8Array | Uint8ClampedArray, width: number, height: number, - hole: Uint8Array + hole: Uint8Array, + holeFrac = 0 ): Promise { const maxSide = isMobile() ? 256 : INPAINT_MAX_SIDE; let w = Math.max(8, Math.round(width)); let h = Math.max(8, Math.round(height)); - const scale = Math.min(1, maxSide / Math.max(w, h)); - w = Math.max(8, Math.floor((w * scale) / 8) * 8); - h = Math.max(8, Math.floor((h * scale) / 8) * 8); + if (w % 8 !== 0 || h % 8 !== 0 || w > maxSide || h > maxSide) { + const scale = Math.min(1, maxSide / Math.max(w, h)); + w = Math.max(8, Math.floor((w * scale) / 8) * 8); + h = Math.max(8, Math.floor((h * scale) / 8) * 8); + } const patchKey = (hole as { patchKey?: string }).patchKey; let key: string; @@ -652,10 +864,18 @@ export async function inpaintCrop( key = `${w}x${h}_${holeCount}_${holeHash}`; } - // 1. FAST ENGINE: Pure TypeScript Border-to-Center Inward Ripple Synthesis (~5ms) + // 1. FAST ENGINE: pure TypeScript exemplar transfer — donor search, then directional walk. const localPatch = fastLocalInpaint(rgba, width, height, hole); - if (!isAiEnabled) { + // The router picks the engine per mask, so the toggle says "the model may be used", not "the + // model is used": a dab the fast engine already answers well stays on the machine. Two + // readings send a mask to the server — the tier chart's own (how much of the frame is under + // the brush, how much structure the rim carries) and the one that only the engine knows: + // whether the photo had a patch to copy at all. A wide mask on calm ground passes every + // threshold and still leaves the engine with a flat field, which is a mask the model should + // take (the tier chart's "large flat area": route it to the server, do not fill it locally). + const needsModel = aiTierNeeded(rgba, width, height, hole, holeFrac) || localFillWasFlat(); + if (!isAiEnabled || !needsModel) { if (patchCache.has(key)) { return patchCache.get(key)!.slice(); } @@ -674,7 +894,12 @@ export async function inpaintCrop( } try { - const { src, hole: mask } = cropPlate(rgba, width, height, hole, w, h); + const { src, hole: plainMask } = cropPlate(rgba, width, height, hole, w, h); + // §8.2 of the hybrid design: the stroke stops at the object and the object's shadow stays, so + // the mask handed to the model is grown 3-5 source pixels. The composite below still feathers + // at the stroke's own radius, so a wider mask is context, not a wider wound. + const grow = Math.max(1, Math.min(5, Math.round(3 * (w / Math.max(1, width))))); + const mask = dilateMask(plainMask, w, h, grow); const srcB64 = uint8ToBase64(src); const maskB64 = uint8ToBase64(mask); @@ -694,6 +919,9 @@ export async function inpaintCrop( const json = await res.json().catch(() => null); if (json && json.ok && json.patch) { const aiPatch = base64ToUint8(json.patch); + if (aiPatch.length === 4 * w * h) { + matchGrain(aiPatch, src, mask, w, h); + } if (patchCache.size > 50) { const first = patchCache.keys().next().value; if (first) patchCache.delete(first); diff --git a/docker/frontend/src/styles/app.css b/docker/frontend/src/styles/app.css index 2b85b83..0f0475d 100644 --- a/docker/frontend/src/styles/app.css +++ b/docker/frontend/src/styles/app.css @@ -3056,6 +3056,52 @@ form[data-key='immich-modal'] { box-sizing: border-box !important; border-right: none !important; } + /* Two things a desktop column is and a phone strip is not. Its height is the + frame's (`.col` above), so a strip that holds one row of chips stood 46vh + tall with that row floating in the middle of the empty bar. And every chip + in it takes the column's own width (`.col .chip` above), which in a row is + the row's width read once per chip: each chip came out as wide as the whole + bar, so the strip was one chip wide and eight chips long, the rest of them + off-screen, and what the visitor saw of a chip was its label alone in an + empty pill. A chip here is as wide as its own label, and a strip as tall as + the chips it holds. */ + .workspace.is-mobile .col, + .workspace.is-mobile .col-main, + .workspace.is-mobile .col-sub, + .workspace.is-mobile .col[data-col="slider"], + .workspace.is-mobile .col-create, + .workspace.is-mobile .col-slider { + height: auto !important; + } + .workspace.is-mobile .col .chip { + width: auto !important; + } + /* The roll, in the bar's own system rather than the stage's. The chip bar + insets itself by 8px, spaces its chips 6px apart, and a chip is a pill on + `--bg-elev`; the strip already sat on that same 8px edge, but it wore the + stage's square frame and floated in the stage's centred cluster, so the row + of frames read as a foreign row hovering above the bar instead of the bar's + own first line. Same gap, same pill, and anchored to the bottom: the roll + ends up the row above the toolbar, which is the row above the chip bar, with + the photo giving way above it. Frames are 44px so the eight of a folder fit + the 414px a phone has between the insets and the roll does not have to be + scrolled to be read. */ + .workspace.is-mobile .studio-strip { + gap: 6px !important; + /* The strip is centred when it is narrower than the stage, and a centred row + starts where its own width leaves it — 18px in, against the bar's 8px. It + takes the whole width so its first frame lands on the bar's own inset. */ + width: 100% !important; + margin-top: auto !important; + padding-bottom: 0 !important; + scroll-padding-inline: 8px !important; + } + .workspace.is-mobile .strip-frame { + width: 44px !important; + height: 44px !important; + border-radius: 999px !important; + background: var(--bg-elev) !important; + } .workspace.is-mobile .col-main { flex-direction: row !important; align-items: center !important; diff --git a/docker/frontend/src/styles/tokens.css b/docker/frontend/src/styles/tokens.css index c7cb219..b8b9f43 100644 --- a/docker/frontend/src/styles/tokens.css +++ b/docker/frontend/src/styles/tokens.css @@ -86,6 +86,11 @@ /* --- base elements ------------------------------------------------------ */ * { box-sizing: border-box; } html, body, #root { height: 100%; margin: 0; } +/* iOS Safari inflates the text of any block wider than the viewport (the phone's + chip strips are `width: max-content` under .col-main, so a selected chip can + blow up to ~3x and take the whole bar). Every size in this app is written in + px, so the boost is off: 100% keeps the sizes as declared. */ +html { -webkit-text-size-adjust: 100%; text-size-adjust: 100%; } body { background: var(--bg); color: var(--text); diff --git a/docker/frontend/src/ui/ImageStage.tsx b/docker/frontend/src/ui/ImageStage.tsx index bb5b8ad..13c3f17 100644 --- a/docker/frontend/src/ui/ImageStage.tsx +++ b/docker/frontend/src/ui/ImageStage.tsx @@ -1150,7 +1150,17 @@ export function ImageStage({ const selBox = box && selSpot !== null && selSpot < brushSpots.length ? (() => { - const s = brushSpots[selSpot]; + const targetStroke = brushSpots[selSpot]?.stroke; + let anchorSpot = brushSpots[selSpot]; + if (typeof targetStroke === 'number') { + for (let i = 0; i < brushSpots.length; i++) { + const sp = brushSpots[i]; + if (sp.stroke === targetStroke && sp.y < anchorSpot.y) { + anchorSpot = sp; + } + } + } + const s = anchorSpot; const off = (s.r * box.width + 14 / view.s) * Math.SQRT1_2; return { left: `calc(${s.x * 100}% + ${off}px)`, diff --git a/docker/recipescam-images.tar.gz.sha256 b/docker/recipescam-images.tar.gz.sha256 index b6259d8..cbddcce 100644 --- a/docker/recipescam-images.tar.gz.sha256 +++ b/docker/recipescam-images.tar.gz.sha256 @@ -1 +1 @@ -6d5c70d163ece1a8cc3225dac50848fbbafd65820bcafbaac3ecb5cca7d07ca2 recipescam-images.tar.gz +7cd97bf92bc1bf2446e53c10ff3e486b94e9ce0f841ebf9ba6a6fbaf81ef79ae recipescam-images.tar.gz