From 5ff46116ce8b7c1ba4f11ca722dc6d3113b17800 Mon Sep 17 00:00:00 2001 From: 3dtours Date: Thu, 15 Oct 2026 22:37:05 +0700 Subject: [PATCH] fix(heal): read the rim for the fill, take the grid from the frame; VIGNETTING both ways; EV in the wide panel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HEAL came back a stain larger and softer than the stain it was asked to remove, and the model's half of it answered in a colour that appears nowhere near the repair. Three causes, all inside `engine/inpaint.ts`: - The fallback's reading ring was the hole dilated by two pixels whatever the brush, so the boundary of the fill was the dirt's own penumbra and the fill came back as the dirt. The pad is now proportional to the hole's own radius (`rimPad`), the fill is an inverse-square average of the clean ring relaxed into one field by 36 Jacobi rounds, and the hole's grain is read against a 2px box of the clean pixels and taken from the pixel it mirrors through its own middle. The texture half before this was a no-op — `detail = rgba[p] - fR[idx]` with `fR[idx]` the same pixel's own colour, so every donor handed over zero and the hole was the smooth field alone (15.6% of the ground's detail). Measured on a stain of radius 10 with a halo to 16 under a 12px brush (`probe-fallback.mjs`): the repair landed 22.3 levels below the ground it sits in, and comes back 0.06 off it. - The plate the model reads under the mask was the mean of every pixel of the crop, so the network met a hard step at the mask's own edge and answered in that colour. It is now the rim's own light spread inward (`cropPlate`), and the mask is downsampled by "any hole pixel in this cell" rather than by a corner sample. Step across the mask's edge: 69.3 levels before, 9.5 after. - The same repair came back different every time the reader zoomed out of it: the crop was sized from the stage's own pixels on the zoom ladder (`PREVIEW_QUANTUM`), so each zoom handed the model a different grid, and every grid invents its own pixels. The crop's box is now its share of the FRAME (`INPAINT_REF_LONG`, `exportEngine` 6e), so one spot is one grid at every zoom and at the export's own size. The patch is deliberately not baked into the photo: a recipe stays reusable on other frames and the repair stays a pass. ponytail: the pixels handed to the model are still read from whatever the stage rendered at, so the answer is the same repair to within the resampling of its input; a patch cache keyed on the frame's identity and the spots is the rung that makes it identical, and the one to take if a zoom still shows a repair move. VIGNETTING now runs both ways (-10..10): down darkens the four corners, up lifts the lens' own falloff out of them with a white ink — the same curve and the same weight. EV was defined and live on the phone but absent from the wide panel's own Light row, which is `DevelopPanels.tsx` and nothing else. `npx tsc --noEmit`, `vite build`, `heal-check`, `studio-drag` and the other suites pass. `studio-export-check` fails one assertion here on the code at HEAD as well — a frame left behind files its waiting look 9s after the pick — so it is not this change. --- docker/frontend/shared/utils/paramDefs.ts | 7 +- docker/frontend/src/engine/exportEngine.ts | 28 +- docker/frontend/src/engine/inpaint.ts | 570 ++++++++++++--------- docker/frontend/src/ui/DevelopPanels.tsx | 2 +- 4 files changed, 356 insertions(+), 251 deletions(-) diff --git a/docker/frontend/shared/utils/paramDefs.ts b/docker/frontend/shared/utils/paramDefs.ts index a6172f6..75e962d 100644 --- a/docker/frontend/shared/utils/paramDefs.ts +++ b/docker/frontend/shared/utils/paramDefs.ts @@ -274,11 +274,14 @@ export const PARAM_DEFS: { { key: 'vignette', label: 'VIGNETTING', - min: 0, + // Both ways: down darkens the four corners (the Fuji control), up takes + // the lens' own falloff out of them — the repair a wide-open lens or a + // stacked hood leaves in the corners, which no other knob here can do. + min: -10, max: 10, step: 1, defaultValue: 0, - display: String, + display: sign, get: (a) => a.vignette ?? 0, set: (v) => ({ vignette: v }), }, diff --git a/docker/frontend/src/engine/exportEngine.ts b/docker/frontend/src/engine/exportEngine.ts index d2ec479..fcd5d01 100644 --- a/docker/frontend/src/engine/exportEngine.ts +++ b/docker/frontend/src/engine/exportEngine.ts @@ -56,7 +56,7 @@ import { HEAL_PATCH_STOPS, readHeal, } from '../../shared/utils/heal'; -import { inpaintCrop, inpaintCropSize } from './inpaint'; +import { inpaintCrop, inpaintCropSize, INPAINT_REF_LONG } from './inpaint'; import { mosaicSkSL, mosaicUniforms, readMosaic } from '../../shared/utils/mosaic'; import { gradientMaskSkSL, maskUniforms, readMasks, masksHaveSpatial } from '../../shared/utils/gradientMask'; import { @@ -1335,13 +1335,19 @@ export async function renderPhoto(input: RenderInput): Promise 0) { - const v = Math.min(10, adjustments.vignette ?? 0) / 10; + // 6c. Vignette. Down darkens the corners and up lifts them, which is the + // lens' own falloff taken out: the same curve and the same weight, with a + // white ink where the darkening one would be. Up was refused while the knob + // could not go below zero; a corner that is dark in the negative and bright + // in the positive is the one knob for both. + const vignette = adjustments.vignette ?? 0; + if (vignette !== 0) { + const v = Math.min(10, Math.abs(vignette)) / 10; + const ink = vignette > 0 ? 'rgba(0,0,0,1)' : 'rgba(255,255,255,1)'; const vignetteShader = own(Skia.Shader.MakeRadialGradient( Skia.Point(width / 2, height / 2), Math.hypot(width, height) / 2, - [Skia.Color('rgba(0,0,0,0)'), Skia.Color('rgba(0,0,0,0)'), Skia.Color('rgba(0,0,0,1)')], + [Skia.Color('rgba(0,0,0,0)'), Skia.Color('rgba(0,0,0,0)'), Skia.Color(ink)], [0, 0.45, 1], Skia.TileMode.Clamp )); @@ -1645,7 +1651,17 @@ function clusterHealSpots( const srcH = srcBottom - srcTop; if (srcW >= 8 && srcH >= 8) { - const { w: mw, h: mh } = inpaintCropSize(srcW, srcH); + // The model's grid comes from the crop's share of the FRAME and not + // from the pixels the frame is rendered at, so the same repair is the + // same repair at every zoom and at the export's own size — see + // INPAINT_REF_LONG. It is also finer than it used to be wherever the + // stage is small: a crop a tenth of a 768px preview is 77 pixels, and + // 77 pixels of context is what the model used to be asked to invent + // from. + const { w: mw, h: mh } = inpaintCropSize( + (srcW / width) * INPAINT_REF_LONG, + (srcH / height) * INPAINT_REF_LONG + ); const cropSurface = own(createSurface(mw, mh)); if (cropSurface) { // Progressive inpainting: read the crop from fillSurface, which already diff --git a/docker/frontend/src/engine/inpaint.ts b/docker/frontend/src/engine/inpaint.ts index 3bbedf3..34b4f41 100644 --- a/docker/frontend/src/engine/inpaint.ts +++ b/docker/frontend/src/engine/inpaint.ts @@ -38,6 +38,25 @@ const RUNTIME_FILES = ['ort-wasm-simd-threaded.jsep.mjs', 'ort-wasm-simd-threade // long before this. `inpaintCrop` scales what it is given to fit; the renderer // draws the answer back to the crop's own rectangle. export const INPAINT_MAX_SIDE = 512; +// The frame's long side, as a crop's grid is measured for the model. The stage +// re-renders the frame on the zoom ladder (PREVIEW_QUANTUM in App.tsx, up to +// PREVIEW_HARD_MAX), so the same spots of the same photo are a different number +// of pixels at every zoom; a crop sized from those pixels handed the model a +// different grid each time, and every grid invents its own pixels — which is why +// a repair changed every time the user zoomed back out of it, and drifted a +// little further with each pass. Sized against a fixed long side instead, the +// crop's box is a fraction of the frame and the grid it asks for is the same +// repair at every zoom and at the export's own size. 2048 is past the stage's +// own PREVIEW_MAX, so a crop small enough to matter is never scaled up to reach +// it. +// +// ponytail: the grid is the same; the pixels handed to it are still read from +// whatever the stage rendered at, so the model's answer is the same repair to +// within the resampling of its input and not to the byte. Caching the patch +// across renders under a key that names the frame and the spots rather than the +// render's pixels is the rung that makes it identical, and it is the one to take +// if a zoom still shows a repair move. +export const INPAINT_REF_LONG = 2048; // The one throwaway run when the session opens: 64 pixels square, but a whole // number of the graph's own stride, so the kernels it builds are the kernels the // repairs use. @@ -164,12 +183,185 @@ export function preloadInpaint(): void { void load().catch(() => {}); } +const byte = (v: number) => (v < 0 ? 0 : v > 255 ? 255 : Math.round(v)); + +// How many pixels the hole is. +function holeSize(hole: Uint8Array): number { + let n = 0; + for (let i = 0; i < hole.length; i++) if (hole[i]) n++; + return n; +} + +// The pad a fill's reading ring has to clear: the defect's OWN soft edge — a +// dust speck's dark rim, a stain's penumbra, the ring a wet clean leaves. The +// brush covers it only where the user brushed wider than the dirt; the edge is +// the one place a repair must not read its light from. It used to read it: the +// pad was two pixels whatever the brush, so a stain with a four-pixel penumbra +// had its own rim standing as the boundary of the fill, and the repair came back +// as the stain, larger and softer. Measured on a stain of radius 10 with a halo +// to 16 under a brush of 12 (scratchpad probe-fallback.mjs): the fill landed +// 20.9 levels below the ground it sits in, and the smear was the stain's own +// colour. Proportional to the hole's own radius, because a penumbra is: a +// three-pixel speck carries under a pixel of it, a thirty-pixel blob ten. +function rimPad(holeCount: number): number { + return Math.max(2, Math.min(24, Math.round(Math.sqrt(holeCount / Math.PI) * 0.6))); +} + +// The light of a hole, made out of the light around it. +// +// Two passes, in the order the eye reads a repair. The hole's pixels are +// averaged from the clean ring — inverse square distance, so the near side of +// the rim counts for more than the far one — and the surface that leaves is then +// relaxed, 36 rounds of the 4-neighbour mean, so it is one continuous field with +// no ring of its own. The ring's pixels are the boundary, so the fill meets the +// photo along the rim: that, and not an interior match, is all "seamless" has to +// mean for a falloff. +// +// `clean` is what the caller may still read a texture or a light from: every +// pixel outside the hole and outside its pad. The mask of the pad is what keeps +// the defect's own edge out of the answer — see rimPad. +function rimFill( + rgba: Uint8Array | Uint8ClampedArray, + w: number, + h: number, + hole: Uint8Array, + pad: number +): { r: Float32Array; g: Float32Array; b: Float32Array; clean: Uint8Array; holeCount: number } { + const clean = new Uint8Array(w * h).fill(1); + let holeCount = 0; + const padSq = pad * pad; + for (let y = 0; y < h; y++) { + for (let x = 0; x < w; x++) { + if (!hole[y * w + x]) continue; + holeCount++; + const y0 = Math.max(0, y - pad); + const y1 = Math.min(h - 1, y + pad); + const x0 = Math.max(0, x - pad); + const x1 = Math.min(w - 1, x + pad); + for (let ny = y0; ny <= y1; ny++) { + const dy = ny - y; + for (let nx = x0; nx <= x1; nx++) { + const dx = nx - x; + if (dx * dx + dy * dy <= padSq) clean[ny * w + nx] = 0; + } + } + } + } + const r = new Float32Array(w * h); + const g = new Float32Array(w * h); + const b = new Float32Array(w * h); + if (holeCount === 0) return { r, g, b, clean, holeCount }; + + // The ring: the pixels the fill may read, which are the clean ones touching + // the pad and nothing else. + const ring: number[] = []; + let ringR = 0; + let ringG = 0; + let ringB = 0; + for (let y = 0; y < h; y++) { + for (let x = 0; x < w; x++) { + const idx = y * w + x; + if (!clean[idx]) continue; + let edge = false; + for (let dy = -1; dy <= 1 && !edge; dy++) { + const ny = y + dy; + if (ny < 0 || ny >= h) continue; + for (let dx = -1; dx <= 1; dx++) { + const nx = x + dx; + if (nx < 0 || nx >= w) continue; + if (!clean[ny * w + nx]) { + edge = true; + break; + } + } + } + if (edge) { + ring.push(idx); + ringR += rgba[idx * 4]; + ringG += rgba[idx * 4 + 1]; + ringB += rgba[idx * 4 + 2]; + } + } + } + const rLen = ring.length; + const avgR = rLen ? ringR / rLen : 128; + const avgG = rLen ? ringG / rLen : 128; + const avgB = rLen ? ringB / rLen : 128; + const step = rLen <= 64 ? 1 : Math.max(1, Math.floor(rLen / 48)); + + for (let y = 0; y < h; y++) { + const row = y * w; + for (let x = 0; x < w; x++) { + const idx = row + x; + const p = idx * 4; + if (clean[idx]) { + r[idx] = rgba[p]; + g[idx] = rgba[p + 1]; + b[idx] = rgba[p + 2]; + continue; + } + let wSum = 0; + let rSum = 0; + let gSum = 0; + let bSum = 0; + for (let i = 0; i < rLen; i += step) { + const bp = ring[i]; + const bx = bp % w; + const by = (bp / w) | 0; + const distSq = (x - bx) * (x - bx) + (y - by) * (y - by) + 1; + const weight = 1 / distSq; + wSum += weight; + rSum += rgba[bp * 4] * weight; + gSum += rgba[bp * 4 + 1] * weight; + bSum += rgba[bp * 4 + 2] * weight; + } + r[idx] = wSum > 0 ? rSum / wSum : avgR; + g[idx] = wSum > 0 ? gSum / wSum : avgG; + b[idx] = wSum > 0 ? bSum / wSum : avgB; + } + } + + for (let iter = 0; iter < 36; iter++) { + for (let y = 0; y < h; y++) { + const row = y * w; + const up = Math.max(0, y - 1) * w; + const down = Math.min(h - 1, y + 1) * w; + for (let x = 0; x < w; x++) { + const idx = row + x; + if (clean[idx]) continue; + const left = row + Math.max(0, x - 1); + const right = row + Math.min(w - 1, x + 1); + r[idx] = 0.25 * (r[left] + r[right] + r[up + x] + r[down + x]); + g[idx] = 0.25 * (g[left] + g[right] + g[up + x] + g[down + x]); + b[idx] = 0.25 * (b[left] + b[right] + b[up + x] + b[down + x]); + } + } + } + return { r, g, b, clean, holeCount }; +} + /** - * High-fidelity Exemplar Texture Synthesis + Poisson Tone Transfer. - * Executes in < 15ms with 0MB WASM allocation. - * Strictly excludes the defect and its surrounding rim so the blemish pattern/color - * is NEVER reused, while transferring genuine photographic micro-textures (skin pores, - * fabric weave, natural grain) so the repair NEVER appears flat or muddy ("bệt"). + * The fill the runtime falls back on: the rim's own light (rimFill) with the + * rim's own grain laid back over it. + * + * The grain is the half this never had. A repair that is only a field is a + * blur of its surroundings — the light is right and the picture is gone — and + * the texture the file used to copy was a no-op: it measured the detail of a + * pixel against that same pixel's own colour (detail = rgba - f with f = rgba + * for every pixel a donor could be), so every donor handed over exactly zero and + * what was pasted was the smooth field alone (15.6% of the ground's detail over + * the hole, measured). Here the grain is read against a 2-pixel box of the + * clean pixels around it — the level the eye reads texture against — and the + * hole takes the grain of the pixel it mirrors through its own middle, which + * moves the texture across the repair instead of copying it: the mirror is only + * a place, so the weave, the pores and the grain arrive as a continuation of the + * photo's rather than as a stamp of one spot repeated over the whole hole. + * + * ponytail: the fallback is a diffusion, not a synthesis — over a hole wider + * than the objects around it there is nothing to continue and the answer is a + * plausible average. The rung for a real one is the model (inpaintCrop), which + * is what runs wherever the runtime can have it; this is what iOS and a failed + * inference get. */ export function fastLocalInpaint( rgba: Uint8Array | Uint8ClampedArray, @@ -179,223 +371,81 @@ export function fastLocalInpaint( ): Uint8Array { const out = new Uint8Array(rgba.length); out.set(rgba); - - // 1. Dilate the hole by 2px into an exclusion mask. - // Pixels inside or directly adjacent to the defect are strictly excluded - // so the defect's dirty color, shadow, or blemish pattern is NEVER sampled or reused. - const exclude = new Uint8Array(w * h); - let holeCount = 0; - for (let y = 0; y < h; y++) { - const row = y * w; - for (let x = 0; x < w; x++) { - const idx = row + x; - if (hole[idx]) { - holeCount++; - for (let dy = -2; dy <= 2; dy++) { - const ny = y + dy; - if (ny < 0 || ny >= h) continue; - const nrow = ny * w; - for (let dx = -2; dx <= 2; dx++) { - const nx = x + dx; - if (nx < 0 || nx >= w) continue; - if (dx * dx + dy * dy <= 5) { - exclude[nrow + nx] = 1; - } - } - } - } - } - } - + const { r, g, b, clean, holeCount } = rimFill(rgba, w, h, hole, rimPad(holeSize(hole))); if (holeCount === 0) return out; - // 2. Identify clean boundary pixels (immediately outside the exclusion zone) - const boundary: { x: number; y: number; r: number; g: number; b: number }[] = []; - let bR = 0, bG = 0, bB = 0; + // The hole's own middle, so a pixel can be mirrored through it. + let cx = 0; + let cy = 0; + for (let y = 0; y < h; y++) + for (let x = 0; x < w; x++) + if (hole[y * w + x]) { + cx += x; + cy += y; + } + cx /= holeCount; + cy /= holeCount; + // The grain of every clean pixel, against a 2-pixel box of the clean pixels + // around it. Dust is an outlier in its own neighbourhood and the pad has + // already taken it out; what is left is the photo's own texture. + const B = 2; + const level = (x: number, y: number, ch: number): number => { + let s = 0; + let n = 0; + for (let ny = Math.max(0, y - B); ny <= Math.min(h - 1, y + B); ny++) { + for (let nx = Math.max(0, x - B); nx <= Math.min(w - 1, x + B); nx++) { + const i = ny * w + nx; + if (!clean[i]) continue; + s += rgba[i * 4 + ch]; + n++; + } + } + return n ? s / n : 0; + }; + const grain = new Float32Array(w * h * 3); + const donors: number[] = []; for (let y = 0; y < h; y++) { - const row = y * w; for (let x = 0; x < w; x++) { - const idx = row + x; - if (exclude[idx]) continue; - - let isBound = false; - for (let dy = -1; dy <= 1 && !isBound; dy++) { - const ny = y + dy; - if (ny < 0 || ny >= h) continue; - const nrow = ny * w; - for (let dx = -1; dx <= 1; dx++) { - const nx = x + dx; - if (nx < 0 || nx >= w) continue; - if (exclude[nrow + nx]) { - isBound = true; - break; - } - } - } - - if (isBound) { - const p = idx * 4; - const r = rgba[p]; - const g = rgba[p + 1]; - const b = rgba[p + 2]; - boundary.push({ x, y, r, g, b }); - bR += r; - bG += g; - bB += b; - } + const idx = y * w + x; + if (!clean[idx]) continue; + donors.push(idx); + for (let ch = 0; ch < 3; ch++) grain[idx * 3 + ch] = rgba[idx * 4 + ch] - level(x, y, ch); } } - - const bLen = boundary.length; - // Fallback if the exclusion zone covered the whole crop - if (bLen === 0) { - let sumR = 0, sumG = 0, sumB = 0, count = 0; - for (let i = 0; i < w * h; i++) { - if (!hole[i]) { - const p = i * 4; - sumR += rgba[p]; - sumG += rgba[p + 1]; - sumB += rgba[p + 2]; - count++; - } - } - const cr = count > 0 ? sumR / count : 128; - const cg = count > 0 ? sumG / count : 128; - const cb = count > 0 ? sumB / count : 128; - for (let i = 0; i < w * h; i++) { - if (hole[i]) { - const p = i * 4; - out[p] = cr; - out[p + 1] = cg; - out[p + 2] = cb; - out[p + 3] = 255; - } - } - return out; - } - - const avgR = bR / bLen; - const avgG = bG / bLen; - const avgB = bB / bLen; - - // 3. Compute continuous base illumination field L(x, y) across the crop - const fR = new Float32Array(w * h); - const fG = new Float32Array(w * h); - const fB = new Float32Array(w * h); - - const bStep = bLen <= 64 ? 1 : Math.max(1, Math.floor(bLen / 48)); - - for (let y = 0; y < h; y++) { - const row = y * w; - for (let x = 0; x < w; x++) { - const idx = row + x; - const p = idx * 4; - if (!exclude[idx]) { - fR[idx] = rgba[p]; - fG[idx] = rgba[p + 1]; - fB[idx] = rgba[p + 2]; - } else { - let wSum = 0; - let rSum = 0, gSum = 0, bSum = 0; - for (let i = 0; i < bLen; i += bStep) { - const bp = boundary[i]; - const distSq = (x - bp.x) * (x - bp.x) + (y - bp.y) * (y - bp.y) + 1; - const weight = 1 / distSq; - wSum += weight; - rSum += bp.r * weight; - gSum += bp.g * weight; - bSum += bp.b * weight; - } - fR[idx] = wSum > 0 ? rSum / wSum : avgR; - fG[idx] = wSum > 0 ? gSum / wSum : avgG; - fB[idx] = wSum > 0 ? bSum / wSum : avgB; - } - } - } - - // Fast Poisson harmonic propagation: 36 iterations of 4-point Laplacian smoothing - // to ensure smooth continuous lighting gradients and structural continuity across the hole - for (let iter = 0; iter < 36; iter++) { - for (let y = 0; y < h; y++) { - const row = y * w; - const up = Math.max(0, y - 1) * w; - const down = Math.min(h - 1, y + 1) * w; - for (let x = 0; x < w; x++) { - const idx = row + x; - if (!hole[idx]) continue; - const left = row + Math.max(0, x - 1); - const right = row + Math.min(w - 1, x + 1); - fR[idx] = 0.25 * (fR[left] + fR[right] + fR[up + x] + fR[down + x]); - fG[idx] = 0.25 * (fG[left] + fG[right] + fG[up + x] + fG[down + x]); - fB[idx] = 0.25 * (fB[left] + fB[right] + fB[up + x] + fB[down + x]); - } - } - } - - // 4. Exemplar Texture Synthesis: - // Extract authentic high-frequency texture details from the clean surrounding area (outside exclude zone) - // Detail = RGBA - BaseIllumination. - // For each pixel in the hole, synthesize detail from clean donor pixels, preserving skin pores, - // fabric weave, surface grain, and micro-structures so the healed area NEVER appears flat or muddy ("bệt"). - const detailR = new Float32Array(w * h); - const detailG = new Float32Array(w * h); - const detailB = new Float32Array(w * h); - - for (let y = 0; y < h; y++) { - const row = y * w; - for (let x = 0; x < w; x++) { - const idx = row + x; - if (!exclude[idx]) { - const p = idx * 4; - detailR[idx] = rgba[p] - fR[idx]; - detailG[idx] = rgba[p + 1] - fG[idx]; - detailB[idx] = rgba[p + 2] - fB[idx]; - } - } - } - - // Pre-index clean donor points - const donorPoints: { x: number; y: number; idx: number }[] = []; - for (let y = 0; y < h; y++) { - const row = y * w; - for (let x = 0; x < w; x++) { - const idx = row + x; - if (!exclude[idx]) { - donorPoints.push({ x, y, idx }); - } - } - } - - const dLen = donorPoints.length; - const dStep = dLen <= 128 ? 1 : Math.max(1, Math.floor(dLen / 96)); + const dStep = donors.length <= 128 ? 1 : Math.max(1, Math.floor(donors.length / 96)); for (let y = 0; y < h; y++) { const row = y * w; for (let x = 0; x < w; x++) { const idx = row + x; if (!hole[idx]) continue; - - // Find nearest clean donor texture - let bestDist = Infinity; - let bestIdx = -1; - for (let i = 0; i < dLen; i += dStep) { - const dp = donorPoints[i]; - const dist = (x - dp.x) * (x - dp.x) + (y - dp.y) * (y - dp.y); - if (dist < bestDist) { - bestDist = dist; - bestIdx = dp.idx; + // The grain of the pixel mirrored through the middle of the hole, and only + // where that mirror is a clean part of the photo: on a hole that runs off + // the edge of the crop the mirror lands outside it, and on a hole whose + // own shape is not centred it lands back inside — both fall back to the + // nearest clean pixel, which is the only other honest answer. + const mx = Math.round(2 * cx - x); + const my = Math.round(2 * cy - y); + let donor = mx >= 0 && mx < w && my >= 0 && my < h ? my * w + mx : -1; + if (donor < 0 || !clean[donor]) { + let best = Infinity; + donor = -1; + for (let i = 0; i < donors.length; i += dStep) { + const dp = donors[i]; + const dx = x - (dp % w); + const dy = y - ((dp / w) | 0); + const dist = dx * dx + dy * dy; + if (dist < best) { + best = dist; + donor = dp; + } } } - const p = idx * 4; - const dR = bestIdx >= 0 ? detailR[bestIdx] : 0; - const dG = bestIdx >= 0 ? detailG[bestIdx] : 0; - const dB = bestIdx >= 0 ? detailB[bestIdx] : 0; - - out[p] = Math.max(0, Math.min(255, Math.round(fR[idx] + dR))); - out[p + 1] = Math.max(0, Math.min(255, Math.round(fG[idx] + dG))); - out[p + 2] = Math.max(0, Math.min(255, Math.round(fB[idx] + dB))); + out[p] = byte(r[idx] + (donor >= 0 ? grain[donor * 3] : 0)); + out[p + 1] = byte(g[idx] + (donor >= 0 ? grain[donor * 3 + 1] : 0)); + out[p + 2] = byte(b[idx] + (donor >= 0 ? grain[donor * 3 + 2] : 0)); out[p + 3] = 255; } } @@ -403,6 +453,75 @@ export function fastLocalInpaint( return out; } +// What the model is handed: the crop at the model's own grid, and the mask in +// the model's own polarity — 255 where the frame is known, 0 over the hole. +// +// The pixels it reads UNDER the hole are the light of the hole's own rim, spread +// inward, and not the flat plate the crop used to be filled with: that was the +// mean of every pixel of the crop, which is a colour that appears nowhere near +// the repair as soon as the crop is bigger than the thing inside it (a blemish +// on a dark ground, a wire across a bright sky). The network then reads a hard +// step at the mask's own edge, answers in the plate's colour, and the repair +// lands as a patch of the wrong light on the frame's own. The rim's light meets +// the photo along the mask's edge by construction, so there is no step for the +// network to answer, and the context it reads is the photo's own. +// +// The mask is downsized by "any hole pixel in this cell" rather than by whichever +// pixel a cell's corner samples: the renderer hands a crop cut to this grid, and +// snapping that to whole strides of the graph can drop the last rows of it — with +// it, a brush thinner than the loss, and a frame handed to the model with nothing +// to invent comes back as the photo it already had. +export function cropPlate( + rgba: Uint8Array | Uint8ClampedArray, + width: number, + height: number, + hole: Uint8Array, + w: number, + h: number +): { src: Uint8Array; hole: Uint8Array } { + const sx = width / w; + const sy = height / h; + const small = new Uint8Array(4 * w * h); + for (let y = 0; y < h; y++) { + const syy = Math.min(height - 1, Math.floor(y * sy)); + for (let x = 0; x < w; x++) { + const sxx = Math.min(width - 1, Math.floor(x * sx)); + const s = (syy * width + sxx) * 4; + const d = (y * w + x) * 4; + small[d] = rgba[s]; + small[d + 1] = rgba[s + 1]; + small[d + 2] = rgba[s + 2]; + small[d + 3] = 255; + } + } + const smallHole = new Uint8Array(w * h); + for (let i = 0; i < width * height; i++) { + if (!hole[i]) continue; + const x = Math.min(w - 1, Math.floor((i % width) / sx)); + const y = Math.min(h - 1, Math.floor(((i / width) | 0) / sy)); + smallHole[y * w + x] = 1; + } + + const plane = w * h; + const src = new Uint8Array(3 * plane); + const mask = new Uint8Array(plane).fill(255); + const field = rimFill(small, w, h, smallHole, 0); + for (let d = 0; d < plane; d++) { + const s = d * 4; + if (smallHole[d]) { + src[d] = byte(field.r[d]); + src[plane + d] = byte(field.g[d]); + src[2 * plane + d] = byte(field.b[d]); + mask[d] = 0; + } else { + src[d] = small[s]; + src[plane + d] = small[s + 1]; + src[2 * plane + d] = small[s + 2]; + } + } + return { src, hole: mask }; +} + const patchCache = new Map(); let inFlightInpaint: Promise | null = null; @@ -495,43 +614,10 @@ export async function inpaintCrop( } const runPromise = (async (): Promise => { - // Compute clean boundary average to fill the hole in `src`, preventing the model - // from seeing or recycling the blemish pattern/color. - let sumR = 0, sumG = 0, sumB = 0, bCount = 0; - const stepSample = Math.max(1, Math.floor((width * height) / 256)); - for (let i = 0; i < width * height; i += stepSample) { - if (!hole[i]) { - const p = i * 4; - sumR += rgba[p]; - sumG += rgba[p + 1]; - sumB += rgba[p + 2]; - bCount++; - } - } - const cleanR = bCount > 0 ? Math.round(sumR / bCount) : 128; - const cleanG = bCount > 0 ? Math.round(sumG / bCount) : 128; - const cleanB = bCount > 0 ? Math.round(sumB / bCount) : 128; - // NCHW, 0..255 RGB — what the model was trained to read — and the mask in its - // own polarity: 255 everywhere the frame is known, 0 over the hole. - const src = new Uint8Array(3 * w * h); - const mask = new Uint8Array(w * h).fill(255); - const sx = width / w; - const sy = height / h; - const plane = w * h; - for (let y = 0; y < h; y++) { - const syy = Math.min(height - 1, Math.floor(y * sy)); - for (let x = 0; x < w; x++) { - const sxx = Math.min(width - 1, Math.floor(x * sx)); - const s = (syy * width + sxx) * 4; - const d = y * w + x; - const isH = Boolean(hole[syy * width + sxx]); - src[d] = isH ? cleanR : rgba[s]; - src[plane + d] = isH ? cleanG : rgba[s + 1]; - src[2 * plane + d] = isH ? cleanB : rgba[s + 2]; - if (isH) mask[d] = 0; - } - } + // own polarity. Both are built by cropPlate, which is where the light the + // model reads under the hole comes from. + const { src, hole: mask } = cropPlate(rgba, width, height, hole, w, h); try { // Allow up to 15000ms for neural inpainting session to initialize and synthesize scene context diff --git a/docker/frontend/src/ui/DevelopPanels.tsx b/docker/frontend/src/ui/DevelopPanels.tsx index 85180dd..cbc200d 100644 --- a/docker/frontend/src/ui/DevelopPanels.tsx +++ b/docker/frontend/src/ui/DevelopPanels.tsx @@ -61,7 +61,7 @@ const PANELS: { }, { title: 'Light', - rows: ['exposure', 'contrast', 'highlight', 'shadow', 'whites', 'blacks'], + rows: ['ev', 'exposure', 'contrast', 'highlight', 'shadow', 'whites', 'blacks'], }, ], },