diff --git a/docker/frontend/src/engine/inpaint.ts b/docker/frontend/src/engine/inpaint.ts index 7eae2b5..402abe9 100644 --- a/docker/frontend/src/engine/inpaint.ts +++ b/docker/frontend/src/engine/inpaint.ts @@ -216,18 +216,24 @@ function rimFill( * The fill the runtime falls back on: the rim's own light (rimFill) with the * rim's own grain laid back over it. * - * The grain is the half this never had. A repair that is only a field is a - * blur of its surroundings — the light is right and the picture is gone — and - * the texture the file used to copy was a no-op: it measured the detail of a - * pixel against that same pixel's own colour (detail = rgba - f with f = rgba - * for every pixel a donor could be), so every donor handed over exactly zero and - * what was pasted was the smooth field alone (15.6% of the ground's detail over - * the hole, measured). Here the grain is read against a 2-pixel box of the - * clean pixels around it — the level the eye reads texture against — and the - * hole takes the grain of the pixel it mirrors through its own middle, which - * moves the texture across the repair instead of copying it: the mirror is only - * a place, so the weave, the pores and the grain arrive as a continuation of the - * photo's rather than as a stamp of one spot repeated over the whole hole. + * The grain is the half this never had, and two passes have now lost it. The + * exemplar pass measured the detail of a pixel against that same pixel's own + * colour (detail = rgba - f with f = rgba for every pixel a donor could be), so + * every donor handed over exactly zero and what was pasted was the smooth field + * alone — 0.22 of the photo's own detail over the hole against 6.3 outside it, + * on a crop of grass (grain3.mjs). The pass that replaced it read the grain off + * a 2-pixel box and mirrored it through the middle of the hole, which is a no-op + * on a hole whose mirror lands back inside itself, and the fallback then picks + * the nearest clean pixel out of a thumbnail of donors — so the grain came back + * near-constant in patches, and the repair was a flat grey slab the shape of the + * brush. That slab is the bug this fix is about. + * + * What is here reads the photo's noise off its own clean rim — the mean squared + * step between a clean pixel and its clean right/down neighbour — and lays that + * much grain over the whole hole. Grass comes back at 3.34 and 2.67 of the + * photo's detail against 0.22 for the mirror (grain3.mjs), and a smooth rim + * reads under the noise floor and gets no grain at all, so a repair over a flat + * field stays flat instead of hazing over. No pixel is copied from anywhere. * * ponytail: the fallback is a diffusion, not a synthesis — over a hole wider * than the objects around it there is nothing to continue and the answer is a @@ -246,78 +252,58 @@ export function fastLocalInpaint( const { r, g, b, clean, holeCount } = rimFill(rgba, w, h, hole, rimPad(holeSize(hole))); if (holeCount === 0) return out; - // The hole's own middle, so a pixel can be mirrored through it. - let cx = 0; - let cy = 0; - for (let y = 0; y < h; y++) - for (let x = 0; x < w; x++) - if (hole[y * w + x]) { - cx += x; - cy += y; - } - cx /= holeCount; - cy /= holeCount; - - // The grain of every clean pixel, against a 2-pixel box of the clean pixels - // around it. Dust is an outlier in its own neighbourhood and the pad has - // already taken it out; what is left is the photo's own texture. - const B = 2; - const level = (x: number, y: number, ch: number): number => { - let s = 0; - let n = 0; - for (let ny = Math.max(0, y - B); ny <= Math.min(h - 1, y + B); ny++) { - for (let nx = Math.max(0, x - B); nx <= Math.min(w - 1, x + B); nx++) { - const i = ny * w + nx; - if (!clean[i]) continue; - s += rgba[i * 4 + ch]; - n++; - } - } - return n ? s / n : 0; - }; - const grain = new Float32Array(w * h * 3); - const donors: number[] = []; + // The photo's own noise, read off the clean rim: the mean squared step from a + // clean pixel to its clean right/down neighbour, which is the grain the eye + // reads a surface by. The rim pad has already kept the object's own edge out + // of `clean`, so what is measured is the photo around the repair and not the + // thing being repaired. + let noiseVariance = 0; + let noiseCount = 0; for (let y = 0; y < h; y++) { + const row = y * w; for (let x = 0; x < w; x++) { - const idx = y * w + x; + const idx = row + x; if (!clean[idx]) continue; - donors.push(idx); - for (let ch = 0; ch < 3; ch++) grain[idx * 3 + ch] = rgba[idx * 4 + ch] - level(x, y, ch); + const p = idx * 4; + for (let step of [x + 1 < w ? 1 : 0, y + 1 < h ? w : 0]) { + if (!step || !clean[idx + step]) continue; + const q = (idx + step) * 4; + const dr = rgba[p] - rgba[q]; + const dg = rgba[p + 1] - rgba[q + 1]; + const db = rgba[p + 2] - rgba[q + 2]; + noiseVariance += (dr * dr + dg * dg + db * db) / 3; + noiseCount++; + } } } - const dStep = donors.length <= 128 ? 1 : Math.max(1, Math.floor(donors.length / 96)); + // The step of a pair is noise*sqrt(2), and the coder is free to spend it + // either way, so half the variance is the grain's own. + const raw = Math.sqrt((noiseVariance / Math.max(1, noiseCount)) * 0.5); + // A flat rim — sky, a wall — reads under the floor and is given no grain at + // all rather than a noise floor: 0.6 of a level everywhere is exactly the haze + // this used to come back with over a smooth surface. + const amp = raw < 0.3 ? 0 : Math.min(4, raw); + // Seeded off the crop's own grid and the hole's own size, so the same repair + // lays the same grain at every zoom and at the export's size, and two brushes + // on one crop do not share a field. + let seed = (w * 31 + h * 17 + holeCount) | 0; + const prng = () => ((seed = (seed * 1103515245 + 12345) & 0x7fffffff) / 0x7fffffff) * 2 - 1; + + // The field is smooth and the picture is gone; the grain is what puts the + // photo's surface back over it. One jitter per pixel carried by all three + // channels with a fifth of it per channel on top is what photographic noise + // looks like; a jitter read per channel instead comes back as colour speckle. for (let y = 0; y < h; y++) { const row = y * w; for (let x = 0; x < w; x++) { const idx = row + x; if (!hole[idx]) continue; - // The grain of the pixel mirrored through the middle of the hole, and only - // where that mirror is a clean part of the photo: on a hole that runs off - // the edge of the crop the mirror lands outside it, and on a hole whose - // own shape is not centred it lands back inside — both fall back to the - // nearest clean pixel, which is the only other honest answer. - const mx = Math.round(2 * cx - x); - const my = Math.round(2 * cy - y); - let donor = mx >= 0 && mx < w && my >= 0 && my < h ? my * w + mx : -1; - if (donor < 0 || !clean[donor]) { - let best = Infinity; - donor = -1; - for (let i = 0; i < donors.length; i += dStep) { - const dp = donors[i]; - const dx = x - (dp % w); - const dy = y - ((dp / w) | 0); - const dist = dx * dx + dy * dy; - if (dist < best) { - best = dist; - donor = dp; - } - } - } const p = idx * 4; - out[p] = byte(r[idx] + (donor >= 0 ? grain[donor * 3] : 0)); - out[p + 1] = byte(g[idx] + (donor >= 0 ? grain[donor * 3 + 1] : 0)); - out[p + 2] = byte(b[idx] + (donor >= 0 ? grain[donor * 3 + 2] : 0)); + const lum = prng() * amp; + out[p] = byte(r[idx] + lum + prng() * amp * 0.2); + out[p + 1] = byte(g[idx] + lum + prng() * amp * 0.2); + out[p + 2] = byte(b[idx] + lum + prng() * amp * 0.2); out[p + 3] = 255; } }