|
|
|
@@ -38,6 +38,25 @@ const RUNTIME_FILES = ['ort-wasm-simd-threaded.jsep.mjs', 'ort-wasm-simd-threade
|
|
|
|
|
// long before this. `inpaintCrop` scales what it is given to fit; the renderer
|
|
|
|
|
// draws the answer back to the crop's own rectangle.
|
|
|
|
|
export const INPAINT_MAX_SIDE = 512;
|
|
|
|
|
// The frame's long side, as a crop's grid is measured for the model. The stage
|
|
|
|
|
// re-renders the frame on the zoom ladder (PREVIEW_QUANTUM in App.tsx, up to
|
|
|
|
|
// PREVIEW_HARD_MAX), so the same spots of the same photo are a different number
|
|
|
|
|
// of pixels at every zoom; a crop sized from those pixels handed the model a
|
|
|
|
|
// different grid each time, and every grid invents its own pixels — which is why
|
|
|
|
|
// a repair changed every time the user zoomed back out of it, and drifted a
|
|
|
|
|
// little further with each pass. Sized against a fixed long side instead, the
|
|
|
|
|
// crop's box is a fraction of the frame and the grid it asks for is the same
|
|
|
|
|
// repair at every zoom and at the export's own size. 2048 is past the stage's
|
|
|
|
|
// own PREVIEW_MAX, so a crop small enough to matter is never scaled up to reach
|
|
|
|
|
// it.
|
|
|
|
|
//
|
|
|
|
|
// ponytail: the grid is the same; the pixels handed to it are still read from
|
|
|
|
|
// whatever the stage rendered at, so the model's answer is the same repair to
|
|
|
|
|
// within the resampling of its input and not to the byte. Caching the patch
|
|
|
|
|
// across renders under a key that names the frame and the spots rather than the
|
|
|
|
|
// render's pixels is the rung that makes it identical, and it is the one to take
|
|
|
|
|
// if a zoom still shows a repair move.
|
|
|
|
|
export const INPAINT_REF_LONG = 2048;
|
|
|
|
|
// The one throwaway run when the session opens: 64 pixels square, but a whole
|
|
|
|
|
// number of the graph's own stride, so the kernels it builds are the kernels the
|
|
|
|
|
// repairs use.
|
|
|
|
@@ -164,12 +183,185 @@ export function preloadInpaint(): void {
|
|
|
|
|
void load().catch(() => {});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const byte = (v: number) => (v < 0 ? 0 : v > 255 ? 255 : Math.round(v));
|
|
|
|
|
|
|
|
|
|
// How many pixels the hole is.
|
|
|
|
|
function holeSize(hole: Uint8Array): number {
|
|
|
|
|
let n = 0;
|
|
|
|
|
for (let i = 0; i < hole.length; i++) if (hole[i]) n++;
|
|
|
|
|
return n;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// The pad a fill's reading ring has to clear: the defect's OWN soft edge — a
|
|
|
|
|
// dust speck's dark rim, a stain's penumbra, the ring a wet clean leaves. The
|
|
|
|
|
// brush covers it only where the user brushed wider than the dirt; the edge is
|
|
|
|
|
// the one place a repair must not read its light from. It used to read it: the
|
|
|
|
|
// pad was two pixels whatever the brush, so a stain with a four-pixel penumbra
|
|
|
|
|
// had its own rim standing as the boundary of the fill, and the repair came back
|
|
|
|
|
// as the stain, larger and softer. Measured on a stain of radius 10 with a halo
|
|
|
|
|
// to 16 under a brush of 12 (scratchpad probe-fallback.mjs): the fill landed
|
|
|
|
|
// 20.9 levels below the ground it sits in, and the smear was the stain's own
|
|
|
|
|
// colour. Proportional to the hole's own radius, because a penumbra is: a
|
|
|
|
|
// three-pixel speck carries under a pixel of it, a thirty-pixel blob ten.
|
|
|
|
|
function rimPad(holeCount: number): number {
|
|
|
|
|
return Math.max(2, Math.min(24, Math.round(Math.sqrt(holeCount / Math.PI) * 0.6)));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// The light of a hole, made out of the light around it.
|
|
|
|
|
//
|
|
|
|
|
// Two passes, in the order the eye reads a repair. The hole's pixels are
|
|
|
|
|
// averaged from the clean ring — inverse square distance, so the near side of
|
|
|
|
|
// the rim counts for more than the far one — and the surface that leaves is then
|
|
|
|
|
// relaxed, 36 rounds of the 4-neighbour mean, so it is one continuous field with
|
|
|
|
|
// no ring of its own. The ring's pixels are the boundary, so the fill meets the
|
|
|
|
|
// photo along the rim: that, and not an interior match, is all "seamless" has to
|
|
|
|
|
// mean for a falloff.
|
|
|
|
|
//
|
|
|
|
|
// `clean` is what the caller may still read a texture or a light from: every
|
|
|
|
|
// pixel outside the hole and outside its pad. The mask of the pad is what keeps
|
|
|
|
|
// the defect's own edge out of the answer — see rimPad.
|
|
|
|
|
function rimFill(
|
|
|
|
|
rgba: Uint8Array | Uint8ClampedArray,
|
|
|
|
|
w: number,
|
|
|
|
|
h: number,
|
|
|
|
|
hole: Uint8Array,
|
|
|
|
|
pad: number
|
|
|
|
|
): { r: Float32Array; g: Float32Array; b: Float32Array; clean: Uint8Array; holeCount: number } {
|
|
|
|
|
const clean = new Uint8Array(w * h).fill(1);
|
|
|
|
|
let holeCount = 0;
|
|
|
|
|
const padSq = pad * pad;
|
|
|
|
|
for (let y = 0; y < h; y++) {
|
|
|
|
|
for (let x = 0; x < w; x++) {
|
|
|
|
|
if (!hole[y * w + x]) continue;
|
|
|
|
|
holeCount++;
|
|
|
|
|
const y0 = Math.max(0, y - pad);
|
|
|
|
|
const y1 = Math.min(h - 1, y + pad);
|
|
|
|
|
const x0 = Math.max(0, x - pad);
|
|
|
|
|
const x1 = Math.min(w - 1, x + pad);
|
|
|
|
|
for (let ny = y0; ny <= y1; ny++) {
|
|
|
|
|
const dy = ny - y;
|
|
|
|
|
for (let nx = x0; nx <= x1; nx++) {
|
|
|
|
|
const dx = nx - x;
|
|
|
|
|
if (dx * dx + dy * dy <= padSq) clean[ny * w + nx] = 0;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const r = new Float32Array(w * h);
|
|
|
|
|
const g = new Float32Array(w * h);
|
|
|
|
|
const b = new Float32Array(w * h);
|
|
|
|
|
if (holeCount === 0) return { r, g, b, clean, holeCount };
|
|
|
|
|
|
|
|
|
|
// The ring: the pixels the fill may read, which are the clean ones touching
|
|
|
|
|
// the pad and nothing else.
|
|
|
|
|
const ring: number[] = [];
|
|
|
|
|
let ringR = 0;
|
|
|
|
|
let ringG = 0;
|
|
|
|
|
let ringB = 0;
|
|
|
|
|
for (let y = 0; y < h; y++) {
|
|
|
|
|
for (let x = 0; x < w; x++) {
|
|
|
|
|
const idx = y * w + x;
|
|
|
|
|
if (!clean[idx]) continue;
|
|
|
|
|
let edge = false;
|
|
|
|
|
for (let dy = -1; dy <= 1 && !edge; dy++) {
|
|
|
|
|
const ny = y + dy;
|
|
|
|
|
if (ny < 0 || ny >= h) continue;
|
|
|
|
|
for (let dx = -1; dx <= 1; dx++) {
|
|
|
|
|
const nx = x + dx;
|
|
|
|
|
if (nx < 0 || nx >= w) continue;
|
|
|
|
|
if (!clean[ny * w + nx]) {
|
|
|
|
|
edge = true;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (edge) {
|
|
|
|
|
ring.push(idx);
|
|
|
|
|
ringR += rgba[idx * 4];
|
|
|
|
|
ringG += rgba[idx * 4 + 1];
|
|
|
|
|
ringB += rgba[idx * 4 + 2];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const rLen = ring.length;
|
|
|
|
|
const avgR = rLen ? ringR / rLen : 128;
|
|
|
|
|
const avgG = rLen ? ringG / rLen : 128;
|
|
|
|
|
const avgB = rLen ? ringB / rLen : 128;
|
|
|
|
|
const step = rLen <= 64 ? 1 : Math.max(1, Math.floor(rLen / 48));
|
|
|
|
|
|
|
|
|
|
for (let y = 0; y < h; y++) {
|
|
|
|
|
const row = y * w;
|
|
|
|
|
for (let x = 0; x < w; x++) {
|
|
|
|
|
const idx = row + x;
|
|
|
|
|
const p = idx * 4;
|
|
|
|
|
if (clean[idx]) {
|
|
|
|
|
r[idx] = rgba[p];
|
|
|
|
|
g[idx] = rgba[p + 1];
|
|
|
|
|
b[idx] = rgba[p + 2];
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
let wSum = 0;
|
|
|
|
|
let rSum = 0;
|
|
|
|
|
let gSum = 0;
|
|
|
|
|
let bSum = 0;
|
|
|
|
|
for (let i = 0; i < rLen; i += step) {
|
|
|
|
|
const bp = ring[i];
|
|
|
|
|
const bx = bp % w;
|
|
|
|
|
const by = (bp / w) | 0;
|
|
|
|
|
const distSq = (x - bx) * (x - bx) + (y - by) * (y - by) + 1;
|
|
|
|
|
const weight = 1 / distSq;
|
|
|
|
|
wSum += weight;
|
|
|
|
|
rSum += rgba[bp * 4] * weight;
|
|
|
|
|
gSum += rgba[bp * 4 + 1] * weight;
|
|
|
|
|
bSum += rgba[bp * 4 + 2] * weight;
|
|
|
|
|
}
|
|
|
|
|
r[idx] = wSum > 0 ? rSum / wSum : avgR;
|
|
|
|
|
g[idx] = wSum > 0 ? gSum / wSum : avgG;
|
|
|
|
|
b[idx] = wSum > 0 ? bSum / wSum : avgB;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (let iter = 0; iter < 36; iter++) {
|
|
|
|
|
for (let y = 0; y < h; y++) {
|
|
|
|
|
const row = y * w;
|
|
|
|
|
const up = Math.max(0, y - 1) * w;
|
|
|
|
|
const down = Math.min(h - 1, y + 1) * w;
|
|
|
|
|
for (let x = 0; x < w; x++) {
|
|
|
|
|
const idx = row + x;
|
|
|
|
|
if (clean[idx]) continue;
|
|
|
|
|
const left = row + Math.max(0, x - 1);
|
|
|
|
|
const right = row + Math.min(w - 1, x + 1);
|
|
|
|
|
r[idx] = 0.25 * (r[left] + r[right] + r[up + x] + r[down + x]);
|
|
|
|
|
g[idx] = 0.25 * (g[left] + g[right] + g[up + x] + g[down + x]);
|
|
|
|
|
b[idx] = 0.25 * (b[left] + b[right] + b[up + x] + b[down + x]);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return { r, g, b, clean, holeCount };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* High-fidelity Exemplar Texture Synthesis + Poisson Tone Transfer.
|
|
|
|
|
* Executes in < 15ms with 0MB WASM allocation.
|
|
|
|
|
* Strictly excludes the defect and its surrounding rim so the blemish pattern/color
|
|
|
|
|
* is NEVER reused, while transferring genuine photographic micro-textures (skin pores,
|
|
|
|
|
* fabric weave, natural grain) so the repair NEVER appears flat or muddy ("bệt").
|
|
|
|
|
* The fill the runtime falls back on: the rim's own light (rimFill) with the
|
|
|
|
|
* rim's own grain laid back over it.
|
|
|
|
|
*
|
|
|
|
|
* The grain is the half this never had. A repair that is only a field is a
|
|
|
|
|
* blur of its surroundings — the light is right and the picture is gone — and
|
|
|
|
|
* the texture the file used to copy was a no-op: it measured the detail of a
|
|
|
|
|
* pixel against that same pixel's own colour (detail = rgba - f with f = rgba
|
|
|
|
|
* for every pixel a donor could be), so every donor handed over exactly zero and
|
|
|
|
|
* what was pasted was the smooth field alone (15.6% of the ground's detail over
|
|
|
|
|
* the hole, measured). Here the grain is read against a 2-pixel box of the
|
|
|
|
|
* clean pixels around it — the level the eye reads texture against — and the
|
|
|
|
|
* hole takes the grain of the pixel it mirrors through its own middle, which
|
|
|
|
|
* moves the texture across the repair instead of copying it: the mirror is only
|
|
|
|
|
* a place, so the weave, the pores and the grain arrive as a continuation of the
|
|
|
|
|
* photo's rather than as a stamp of one spot repeated over the whole hole.
|
|
|
|
|
*
|
|
|
|
|
* ponytail: the fallback is a diffusion, not a synthesis — over a hole wider
|
|
|
|
|
* than the objects around it there is nothing to continue and the answer is a
|
|
|
|
|
* plausible average. The rung for a real one is the model (inpaintCrop), which
|
|
|
|
|
* is what runs wherever the runtime can have it; this is what iOS and a failed
|
|
|
|
|
* inference get.
|
|
|
|
|
*/
|
|
|
|
|
export function fastLocalInpaint(
|
|
|
|
|
rgba: Uint8Array | Uint8ClampedArray,
|
|
|
|
@@ -179,223 +371,81 @@ export function fastLocalInpaint(
|
|
|
|
|
): Uint8Array {
|
|
|
|
|
const out = new Uint8Array(rgba.length);
|
|
|
|
|
out.set(rgba);
|
|
|
|
|
|
|
|
|
|
// 1. Dilate the hole by 2px into an exclusion mask.
|
|
|
|
|
// Pixels inside or directly adjacent to the defect are strictly excluded
|
|
|
|
|
// so the defect's dirty color, shadow, or blemish pattern is NEVER sampled or reused.
|
|
|
|
|
const exclude = new Uint8Array(w * h);
|
|
|
|
|
let holeCount = 0;
|
|
|
|
|
for (let y = 0; y < h; y++) {
|
|
|
|
|
const row = y * w;
|
|
|
|
|
for (let x = 0; x < w; x++) {
|
|
|
|
|
const idx = row + x;
|
|
|
|
|
if (hole[idx]) {
|
|
|
|
|
holeCount++;
|
|
|
|
|
for (let dy = -2; dy <= 2; dy++) {
|
|
|
|
|
const ny = y + dy;
|
|
|
|
|
if (ny < 0 || ny >= h) continue;
|
|
|
|
|
const nrow = ny * w;
|
|
|
|
|
for (let dx = -2; dx <= 2; dx++) {
|
|
|
|
|
const nx = x + dx;
|
|
|
|
|
if (nx < 0 || nx >= w) continue;
|
|
|
|
|
if (dx * dx + dy * dy <= 5) {
|
|
|
|
|
exclude[nrow + nx] = 1;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const { r, g, b, clean, holeCount } = rimFill(rgba, w, h, hole, rimPad(holeSize(hole)));
|
|
|
|
|
if (holeCount === 0) return out;
|
|
|
|
|
|
|
|
|
|
// 2. Identify clean boundary pixels (immediately outside the exclusion zone)
|
|
|
|
|
const boundary: { x: number; y: number; r: number; g: number; b: number }[] = [];
|
|
|
|
|
let bR = 0, bG = 0, bB = 0;
|
|
|
|
|
// The hole's own middle, so a pixel can be mirrored through it.
|
|
|
|
|
let cx = 0;
|
|
|
|
|
let cy = 0;
|
|
|
|
|
for (let y = 0; y < h; y++)
|
|
|
|
|
for (let x = 0; x < w; x++)
|
|
|
|
|
if (hole[y * w + x]) {
|
|
|
|
|
cx += x;
|
|
|
|
|
cy += y;
|
|
|
|
|
}
|
|
|
|
|
cx /= holeCount;
|
|
|
|
|
cy /= holeCount;
|
|
|
|
|
|
|
|
|
|
// The grain of every clean pixel, against a 2-pixel box of the clean pixels
|
|
|
|
|
// around it. Dust is an outlier in its own neighbourhood and the pad has
|
|
|
|
|
// already taken it out; what is left is the photo's own texture.
|
|
|
|
|
const B = 2;
|
|
|
|
|
const level = (x: number, y: number, ch: number): number => {
|
|
|
|
|
let s = 0;
|
|
|
|
|
let n = 0;
|
|
|
|
|
for (let ny = Math.max(0, y - B); ny <= Math.min(h - 1, y + B); ny++) {
|
|
|
|
|
for (let nx = Math.max(0, x - B); nx <= Math.min(w - 1, x + B); nx++) {
|
|
|
|
|
const i = ny * w + nx;
|
|
|
|
|
if (!clean[i]) continue;
|
|
|
|
|
s += rgba[i * 4 + ch];
|
|
|
|
|
n++;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return n ? s / n : 0;
|
|
|
|
|
};
|
|
|
|
|
const grain = new Float32Array(w * h * 3);
|
|
|
|
|
const donors: number[] = [];
|
|
|
|
|
for (let y = 0; y < h; y++) {
|
|
|
|
|
const row = y * w;
|
|
|
|
|
for (let x = 0; x < w; x++) {
|
|
|
|
|
const idx = row + x;
|
|
|
|
|
if (exclude[idx]) continue;
|
|
|
|
|
|
|
|
|
|
let isBound = false;
|
|
|
|
|
for (let dy = -1; dy <= 1 && !isBound; dy++) {
|
|
|
|
|
const ny = y + dy;
|
|
|
|
|
if (ny < 0 || ny >= h) continue;
|
|
|
|
|
const nrow = ny * w;
|
|
|
|
|
for (let dx = -1; dx <= 1; dx++) {
|
|
|
|
|
const nx = x + dx;
|
|
|
|
|
if (nx < 0 || nx >= w) continue;
|
|
|
|
|
if (exclude[nrow + nx]) {
|
|
|
|
|
isBound = true;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (isBound) {
|
|
|
|
|
const p = idx * 4;
|
|
|
|
|
const r = rgba[p];
|
|
|
|
|
const g = rgba[p + 1];
|
|
|
|
|
const b = rgba[p + 2];
|
|
|
|
|
boundary.push({ x, y, r, g, b });
|
|
|
|
|
bR += r;
|
|
|
|
|
bG += g;
|
|
|
|
|
bB += b;
|
|
|
|
|
}
|
|
|
|
|
const idx = y * w + x;
|
|
|
|
|
if (!clean[idx]) continue;
|
|
|
|
|
donors.push(idx);
|
|
|
|
|
for (let ch = 0; ch < 3; ch++) grain[idx * 3 + ch] = rgba[idx * 4 + ch] - level(x, y, ch);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const bLen = boundary.length;
|
|
|
|
|
// Fallback if the exclusion zone covered the whole crop
|
|
|
|
|
if (bLen === 0) {
|
|
|
|
|
let sumR = 0, sumG = 0, sumB = 0, count = 0;
|
|
|
|
|
for (let i = 0; i < w * h; i++) {
|
|
|
|
|
if (!hole[i]) {
|
|
|
|
|
const p = i * 4;
|
|
|
|
|
sumR += rgba[p];
|
|
|
|
|
sumG += rgba[p + 1];
|
|
|
|
|
sumB += rgba[p + 2];
|
|
|
|
|
count++;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const cr = count > 0 ? sumR / count : 128;
|
|
|
|
|
const cg = count > 0 ? sumG / count : 128;
|
|
|
|
|
const cb = count > 0 ? sumB / count : 128;
|
|
|
|
|
for (let i = 0; i < w * h; i++) {
|
|
|
|
|
if (hole[i]) {
|
|
|
|
|
const p = i * 4;
|
|
|
|
|
out[p] = cr;
|
|
|
|
|
out[p + 1] = cg;
|
|
|
|
|
out[p + 2] = cb;
|
|
|
|
|
out[p + 3] = 255;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const avgR = bR / bLen;
|
|
|
|
|
const avgG = bG / bLen;
|
|
|
|
|
const avgB = bB / bLen;
|
|
|
|
|
|
|
|
|
|
// 3. Compute continuous base illumination field L(x, y) across the crop
|
|
|
|
|
const fR = new Float32Array(w * h);
|
|
|
|
|
const fG = new Float32Array(w * h);
|
|
|
|
|
const fB = new Float32Array(w * h);
|
|
|
|
|
|
|
|
|
|
const bStep = bLen <= 64 ? 1 : Math.max(1, Math.floor(bLen / 48));
|
|
|
|
|
|
|
|
|
|
for (let y = 0; y < h; y++) {
|
|
|
|
|
const row = y * w;
|
|
|
|
|
for (let x = 0; x < w; x++) {
|
|
|
|
|
const idx = row + x;
|
|
|
|
|
const p = idx * 4;
|
|
|
|
|
if (!exclude[idx]) {
|
|
|
|
|
fR[idx] = rgba[p];
|
|
|
|
|
fG[idx] = rgba[p + 1];
|
|
|
|
|
fB[idx] = rgba[p + 2];
|
|
|
|
|
} else {
|
|
|
|
|
let wSum = 0;
|
|
|
|
|
let rSum = 0, gSum = 0, bSum = 0;
|
|
|
|
|
for (let i = 0; i < bLen; i += bStep) {
|
|
|
|
|
const bp = boundary[i];
|
|
|
|
|
const distSq = (x - bp.x) * (x - bp.x) + (y - bp.y) * (y - bp.y) + 1;
|
|
|
|
|
const weight = 1 / distSq;
|
|
|
|
|
wSum += weight;
|
|
|
|
|
rSum += bp.r * weight;
|
|
|
|
|
gSum += bp.g * weight;
|
|
|
|
|
bSum += bp.b * weight;
|
|
|
|
|
}
|
|
|
|
|
fR[idx] = wSum > 0 ? rSum / wSum : avgR;
|
|
|
|
|
fG[idx] = wSum > 0 ? gSum / wSum : avgG;
|
|
|
|
|
fB[idx] = wSum > 0 ? bSum / wSum : avgB;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Fast Poisson harmonic propagation: 36 iterations of 4-point Laplacian smoothing
|
|
|
|
|
// to ensure smooth continuous lighting gradients and structural continuity across the hole
|
|
|
|
|
for (let iter = 0; iter < 36; iter++) {
|
|
|
|
|
for (let y = 0; y < h; y++) {
|
|
|
|
|
const row = y * w;
|
|
|
|
|
const up = Math.max(0, y - 1) * w;
|
|
|
|
|
const down = Math.min(h - 1, y + 1) * w;
|
|
|
|
|
for (let x = 0; x < w; x++) {
|
|
|
|
|
const idx = row + x;
|
|
|
|
|
if (!hole[idx]) continue;
|
|
|
|
|
const left = row + Math.max(0, x - 1);
|
|
|
|
|
const right = row + Math.min(w - 1, x + 1);
|
|
|
|
|
fR[idx] = 0.25 * (fR[left] + fR[right] + fR[up + x] + fR[down + x]);
|
|
|
|
|
fG[idx] = 0.25 * (fG[left] + fG[right] + fG[up + x] + fG[down + x]);
|
|
|
|
|
fB[idx] = 0.25 * (fB[left] + fB[right] + fB[up + x] + fB[down + x]);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 4. Exemplar Texture Synthesis:
|
|
|
|
|
// Extract authentic high-frequency texture details from the clean surrounding area (outside exclude zone)
|
|
|
|
|
// Detail = RGBA - BaseIllumination.
|
|
|
|
|
// For each pixel in the hole, synthesize detail from clean donor pixels, preserving skin pores,
|
|
|
|
|
// fabric weave, surface grain, and micro-structures so the healed area NEVER appears flat or muddy ("bệt").
|
|
|
|
|
const detailR = new Float32Array(w * h);
|
|
|
|
|
const detailG = new Float32Array(w * h);
|
|
|
|
|
const detailB = new Float32Array(w * h);
|
|
|
|
|
|
|
|
|
|
for (let y = 0; y < h; y++) {
|
|
|
|
|
const row = y * w;
|
|
|
|
|
for (let x = 0; x < w; x++) {
|
|
|
|
|
const idx = row + x;
|
|
|
|
|
if (!exclude[idx]) {
|
|
|
|
|
const p = idx * 4;
|
|
|
|
|
detailR[idx] = rgba[p] - fR[idx];
|
|
|
|
|
detailG[idx] = rgba[p + 1] - fG[idx];
|
|
|
|
|
detailB[idx] = rgba[p + 2] - fB[idx];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Pre-index clean donor points
|
|
|
|
|
const donorPoints: { x: number; y: number; idx: number }[] = [];
|
|
|
|
|
for (let y = 0; y < h; y++) {
|
|
|
|
|
const row = y * w;
|
|
|
|
|
for (let x = 0; x < w; x++) {
|
|
|
|
|
const idx = row + x;
|
|
|
|
|
if (!exclude[idx]) {
|
|
|
|
|
donorPoints.push({ x, y, idx });
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const dLen = donorPoints.length;
|
|
|
|
|
const dStep = dLen <= 128 ? 1 : Math.max(1, Math.floor(dLen / 96));
|
|
|
|
|
const dStep = donors.length <= 128 ? 1 : Math.max(1, Math.floor(donors.length / 96));
|
|
|
|
|
|
|
|
|
|
for (let y = 0; y < h; y++) {
|
|
|
|
|
const row = y * w;
|
|
|
|
|
for (let x = 0; x < w; x++) {
|
|
|
|
|
const idx = row + x;
|
|
|
|
|
if (!hole[idx]) continue;
|
|
|
|
|
|
|
|
|
|
// Find nearest clean donor texture
|
|
|
|
|
let bestDist = Infinity;
|
|
|
|
|
let bestIdx = -1;
|
|
|
|
|
for (let i = 0; i < dLen; i += dStep) {
|
|
|
|
|
const dp = donorPoints[i];
|
|
|
|
|
const dist = (x - dp.x) * (x - dp.x) + (y - dp.y) * (y - dp.y);
|
|
|
|
|
if (dist < bestDist) {
|
|
|
|
|
bestDist = dist;
|
|
|
|
|
bestIdx = dp.idx;
|
|
|
|
|
// The grain of the pixel mirrored through the middle of the hole, and only
|
|
|
|
|
// where that mirror is a clean part of the photo: on a hole that runs off
|
|
|
|
|
// the edge of the crop the mirror lands outside it, and on a hole whose
|
|
|
|
|
// own shape is not centred it lands back inside — both fall back to the
|
|
|
|
|
// nearest clean pixel, which is the only other honest answer.
|
|
|
|
|
const mx = Math.round(2 * cx - x);
|
|
|
|
|
const my = Math.round(2 * cy - y);
|
|
|
|
|
let donor = mx >= 0 && mx < w && my >= 0 && my < h ? my * w + mx : -1;
|
|
|
|
|
if (donor < 0 || !clean[donor]) {
|
|
|
|
|
let best = Infinity;
|
|
|
|
|
donor = -1;
|
|
|
|
|
for (let i = 0; i < donors.length; i += dStep) {
|
|
|
|
|
const dp = donors[i];
|
|
|
|
|
const dx = x - (dp % w);
|
|
|
|
|
const dy = y - ((dp / w) | 0);
|
|
|
|
|
const dist = dx * dx + dy * dy;
|
|
|
|
|
if (dist < best) {
|
|
|
|
|
best = dist;
|
|
|
|
|
donor = dp;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const p = idx * 4;
|
|
|
|
|
const dR = bestIdx >= 0 ? detailR[bestIdx] : 0;
|
|
|
|
|
const dG = bestIdx >= 0 ? detailG[bestIdx] : 0;
|
|
|
|
|
const dB = bestIdx >= 0 ? detailB[bestIdx] : 0;
|
|
|
|
|
|
|
|
|
|
out[p] = Math.max(0, Math.min(255, Math.round(fR[idx] + dR)));
|
|
|
|
|
out[p + 1] = Math.max(0, Math.min(255, Math.round(fG[idx] + dG)));
|
|
|
|
|
out[p + 2] = Math.max(0, Math.min(255, Math.round(fB[idx] + dB)));
|
|
|
|
|
out[p] = byte(r[idx] + (donor >= 0 ? grain[donor * 3] : 0));
|
|
|
|
|
out[p + 1] = byte(g[idx] + (donor >= 0 ? grain[donor * 3 + 1] : 0));
|
|
|
|
|
out[p + 2] = byte(b[idx] + (donor >= 0 ? grain[donor * 3 + 2] : 0));
|
|
|
|
|
out[p + 3] = 255;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@@ -403,6 +453,75 @@ export function fastLocalInpaint(
|
|
|
|
|
return out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// What the model is handed: the crop at the model's own grid, and the mask in
|
|
|
|
|
// the model's own polarity — 255 where the frame is known, 0 over the hole.
|
|
|
|
|
//
|
|
|
|
|
// The pixels it reads UNDER the hole are the light of the hole's own rim, spread
|
|
|
|
|
// inward, and not the flat plate the crop used to be filled with: that was the
|
|
|
|
|
// mean of every pixel of the crop, which is a colour that appears nowhere near
|
|
|
|
|
// the repair as soon as the crop is bigger than the thing inside it (a blemish
|
|
|
|
|
// on a dark ground, a wire across a bright sky). The network then reads a hard
|
|
|
|
|
// step at the mask's own edge, answers in the plate's colour, and the repair
|
|
|
|
|
// lands as a patch of the wrong light on the frame's own. The rim's light meets
|
|
|
|
|
// the photo along the mask's edge by construction, so there is no step for the
|
|
|
|
|
// network to answer, and the context it reads is the photo's own.
|
|
|
|
|
//
|
|
|
|
|
// The mask is downsized by "any hole pixel in this cell" rather than by whichever
|
|
|
|
|
// pixel a cell's corner samples: the renderer hands a crop cut to this grid, and
|
|
|
|
|
// snapping that to whole strides of the graph can drop the last rows of it — with
|
|
|
|
|
// it, a brush thinner than the loss, and a frame handed to the model with nothing
|
|
|
|
|
// to invent comes back as the photo it already had.
|
|
|
|
|
export function cropPlate(
|
|
|
|
|
rgba: Uint8Array | Uint8ClampedArray,
|
|
|
|
|
width: number,
|
|
|
|
|
height: number,
|
|
|
|
|
hole: Uint8Array,
|
|
|
|
|
w: number,
|
|
|
|
|
h: number
|
|
|
|
|
): { src: Uint8Array; hole: Uint8Array } {
|
|
|
|
|
const sx = width / w;
|
|
|
|
|
const sy = height / h;
|
|
|
|
|
const small = new Uint8Array(4 * w * h);
|
|
|
|
|
for (let y = 0; y < h; y++) {
|
|
|
|
|
const syy = Math.min(height - 1, Math.floor(y * sy));
|
|
|
|
|
for (let x = 0; x < w; x++) {
|
|
|
|
|
const sxx = Math.min(width - 1, Math.floor(x * sx));
|
|
|
|
|
const s = (syy * width + sxx) * 4;
|
|
|
|
|
const d = (y * w + x) * 4;
|
|
|
|
|
small[d] = rgba[s];
|
|
|
|
|
small[d + 1] = rgba[s + 1];
|
|
|
|
|
small[d + 2] = rgba[s + 2];
|
|
|
|
|
small[d + 3] = 255;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const smallHole = new Uint8Array(w * h);
|
|
|
|
|
for (let i = 0; i < width * height; i++) {
|
|
|
|
|
if (!hole[i]) continue;
|
|
|
|
|
const x = Math.min(w - 1, Math.floor((i % width) / sx));
|
|
|
|
|
const y = Math.min(h - 1, Math.floor(((i / width) | 0) / sy));
|
|
|
|
|
smallHole[y * w + x] = 1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const plane = w * h;
|
|
|
|
|
const src = new Uint8Array(3 * plane);
|
|
|
|
|
const mask = new Uint8Array(plane).fill(255);
|
|
|
|
|
const field = rimFill(small, w, h, smallHole, 0);
|
|
|
|
|
for (let d = 0; d < plane; d++) {
|
|
|
|
|
const s = d * 4;
|
|
|
|
|
if (smallHole[d]) {
|
|
|
|
|
src[d] = byte(field.r[d]);
|
|
|
|
|
src[plane + d] = byte(field.g[d]);
|
|
|
|
|
src[2 * plane + d] = byte(field.b[d]);
|
|
|
|
|
mask[d] = 0;
|
|
|
|
|
} else {
|
|
|
|
|
src[d] = small[s];
|
|
|
|
|
src[plane + d] = small[s + 1];
|
|
|
|
|
src[2 * plane + d] = small[s + 2];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return { src, hole: mask };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const patchCache = new Map<string, Uint8Array>();
|
|
|
|
|
let inFlightInpaint: Promise<Uint8Array | null> | null = null;
|
|
|
|
|
|
|
|
|
@@ -495,43 +614,10 @@ export async function inpaintCrop(
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const runPromise = (async (): Promise<Uint8Array | null> => {
|
|
|
|
|
// Compute clean boundary average to fill the hole in `src`, preventing the model
|
|
|
|
|
// from seeing or recycling the blemish pattern/color.
|
|
|
|
|
let sumR = 0, sumG = 0, sumB = 0, bCount = 0;
|
|
|
|
|
const stepSample = Math.max(1, Math.floor((width * height) / 256));
|
|
|
|
|
for (let i = 0; i < width * height; i += stepSample) {
|
|
|
|
|
if (!hole[i]) {
|
|
|
|
|
const p = i * 4;
|
|
|
|
|
sumR += rgba[p];
|
|
|
|
|
sumG += rgba[p + 1];
|
|
|
|
|
sumB += rgba[p + 2];
|
|
|
|
|
bCount++;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const cleanR = bCount > 0 ? Math.round(sumR / bCount) : 128;
|
|
|
|
|
const cleanG = bCount > 0 ? Math.round(sumG / bCount) : 128;
|
|
|
|
|
const cleanB = bCount > 0 ? Math.round(sumB / bCount) : 128;
|
|
|
|
|
|
|
|
|
|
// NCHW, 0..255 RGB — what the model was trained to read — and the mask in its
|
|
|
|
|
// own polarity: 255 everywhere the frame is known, 0 over the hole.
|
|
|
|
|
const src = new Uint8Array(3 * w * h);
|
|
|
|
|
const mask = new Uint8Array(w * h).fill(255);
|
|
|
|
|
const sx = width / w;
|
|
|
|
|
const sy = height / h;
|
|
|
|
|
const plane = w * h;
|
|
|
|
|
for (let y = 0; y < h; y++) {
|
|
|
|
|
const syy = Math.min(height - 1, Math.floor(y * sy));
|
|
|
|
|
for (let x = 0; x < w; x++) {
|
|
|
|
|
const sxx = Math.min(width - 1, Math.floor(x * sx));
|
|
|
|
|
const s = (syy * width + sxx) * 4;
|
|
|
|
|
const d = y * w + x;
|
|
|
|
|
const isH = Boolean(hole[syy * width + sxx]);
|
|
|
|
|
src[d] = isH ? cleanR : rgba[s];
|
|
|
|
|
src[plane + d] = isH ? cleanG : rgba[s + 1];
|
|
|
|
|
src[2 * plane + d] = isH ? cleanB : rgba[s + 2];
|
|
|
|
|
if (isH) mask[d] = 0;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// own polarity. Both are built by cropPlate, which is where the light the
|
|
|
|
|
// model reads under the hole comes from.
|
|
|
|
|
const { src, hole: mask } = cropPlate(rgba, width, height, hole, w, h);
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
// Allow up to 15000ms for neural inpainting session to initialize and synthesize scene context
|
|
|
|
|