bbc64118a5
- backend: preserve aspect ratio with edge-replicated square padding into 512x512 LaMa model - backend: fix inverted mask threshold (>0.85*plane) and prune CUDA/TensorRT binaries for slim image - frontend: enable server AI by default and route masks >3.5% or edge-crossing to LaMa Tier 2 - frontend: enforce 60%-80% context padding (min 64px) aligned to multiples of 8 - frontend: add 3-5px binary mask dilation and high-frequency sensor grain matching - frontend: anchor deletion badge to topmost point of brush stroke - build & export: updated docker containers and recipescam-images.tar.gz (357MB)
254 lines
13 KiB
JavaScript
254 lines
13 KiB
JavaScript
// The hybrid's three client-side rules, measured on synthetic frames whose answer is known:
|
|
//
|
|
// * the router — which of the two engines a mask goes to (a speck of dust on open ground is the
|
|
// fast engine's job and costs nothing; an object, or a mark that sits across a real edge,
|
|
// goes to the model),
|
|
// * the mask the model is given — grown a few pixels over the stroke so the object's shadow
|
|
// goes with it, and only over the stroke, because the brush is not the wound,
|
|
// * the grain the model leaves behind — a model patch is a clean surface, and a clean surface
|
|
// in a photograph of sensor noise reads as a smudge, so it is given back the ground's own
|
|
// grain and nothing where the ground has none.
|
|
//
|
|
// node scripts/heal-route-check.mjs
|
|
import assert from 'node:assert/strict';
|
|
import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { pathToFileURL } from 'node:url';
|
|
import ts from 'typescript';
|
|
|
|
const SRC = new URL('../src/engine/inpaint.ts', import.meta.url);
|
|
const dir = mkdtempSync(join(tmpdir(), 'heal-route-'));
|
|
writeFileSync(
|
|
join(dir, 'inpaint.mjs'),
|
|
ts.transpileModule(readFileSync(SRC, 'utf8'), {
|
|
compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2022 },
|
|
}).outputText
|
|
);
|
|
const { aiTierNeeded, dilateMask, matchGrain, AI_MAX_MASK_AREA, fastLocalInpaint, localFillWasFlat } = await import(
|
|
pathToFileURL(join(dir, 'inpaint.mjs')).href
|
|
);
|
|
|
|
const W = 512, H = 512;
|
|
const luma = (v) => Math.max(0, Math.min(255, v));
|
|
|
|
// A frame builder: `at(x, y)` says what the ground looks like, and the hole is a disc on it.
|
|
function frame(at, { cx = 256, cy = 256, r = 12, noise = 0, seed = 7 } = {}) {
|
|
const rgba = new Uint8Array(W * H * 4);
|
|
const hole = new Uint8Array(W * H);
|
|
let s = seed;
|
|
const rnd = () => ((s = (s * 1103515245 + 12345) & 0x7fffffff) / 0x7fffffff) - 0.5;
|
|
for (let y = 0; y < H; y++) {
|
|
for (let x = 0; x < W; x++) {
|
|
const i = y * W + x, p = i * 4;
|
|
const v = luma(at(x, y) + (noise ? rnd() * noise : 0));
|
|
rgba[p] = rgba[p + 1] = rgba[p + 2] = v;
|
|
rgba[p + 3] = 255;
|
|
if (r > 0 && (x - cx) ** 2 + (y - cy) ** 2 <= r * r) hole[i] = 1;
|
|
}
|
|
}
|
|
return { rgba, hole };
|
|
}
|
|
|
|
const flat = () => 128;
|
|
const steps = (x) => (x < 128 || x > 380 ? 80 : 176);
|
|
const grain = () => 128;
|
|
const FRAME_AREA = W * H; // this frame is the whole picture for the routing tests
|
|
|
|
// 1. A speck of dust on open ground: the fast engine, no server.
|
|
const speck = frame(flat, { r: 12 });
|
|
assert.equal(aiTierNeeded(speck.rgba, W, H, speck.hole, (Math.PI * 12 * 12) / FRAME_AREA), false);
|
|
|
|
// 2. Something the size of an object: the model. On this 512x512 frame a 60px radius is 4.3% of
|
|
// the area, which is the same order as a 5%-of-width brush on a 3:2 frame.
|
|
const object = frame(flat, { r: 60 });
|
|
assert.equal(aiTierNeeded(object.rgba, W, H, object.hole, (Math.PI * 60 * 60) / FRAME_AREA), true);
|
|
assert.ok(
|
|
(Math.PI * 60 * 60) / FRAME_AREA > AI_MAX_MASK_AREA,
|
|
'the object case must be over the 3.5% the chart draws'
|
|
);
|
|
|
|
// 2b. The brush at its widest on open ground is still a speck by that measure: the fast engine.
|
|
const wideDab = frame(flat, { r: 26 });
|
|
assert.equal(aiTierNeeded(wideDab.rgba, W, H, wideDab.hole, (Math.PI * 26 * 26) / FRAME_AREA), false);
|
|
|
|
// 2c. With no frame to judge the area against, the window is what is left — and a real window is
|
|
// a tight one around its hole, so a mark that fills an eighth of it goes to the model.
|
|
const filled = frame(flat, { r: 100 });
|
|
assert.equal(aiTierNeeded(filled.rgba, W, H, filled.hole), true);
|
|
|
|
// 3. A speck, but one sitting across a real edge: the model, however small it is. The ground here
|
|
// is quiet (rim 1.88 of 255 either side of the line) so the area test would have kept it home;
|
|
// what sends it out is that the rim carries six times the window's own structure.
|
|
const onEdge = frame(steps, { cx: 128, cy: 256, r: 12 });
|
|
assert.equal(aiTierNeeded(onEdge.rgba, W, H, onEdge.hole, (Math.PI * 12 * 12) / FRAME_AREA), true);
|
|
|
|
// 3b. Busy ground — foliage, water ripples, a crowd — is where a copied patch shows up as a slab
|
|
// of somewhere else, so a speck there goes to the model too, even with no edge in sight.
|
|
const foliage = frame((x, y) => 120 + 50 * Math.abs(Math.sin(x * 0.21 + y * 0.13)) + 30 * Math.sin(x * 0.7 + y * 1.3), { r: 26 });
|
|
assert.equal(aiTierNeeded(foliage.rgba, W, H, foliage.hole, (Math.PI * 26 * 26) / FRAME_AREA), true);
|
|
|
|
// 3c. ...but ripples are not that busy, and the engine does answer them: a speck on water stays.
|
|
const water = frame((x, y) => 128 + 18 * Math.sin(x * 0.35 + y * 0.5) + 10 * Math.sin(x * 0.9 - y * 0.7), { r: 26 });
|
|
assert.equal(aiTierNeeded(water.rgba, W, H, water.hole, (Math.PI * 26 * 26) / FRAME_AREA), false);
|
|
|
|
// 4. A speck on ground that is nothing but sensor grain: the fast engine. Grain is not structure.
|
|
const noisy = frame(grain, { r: 12, noise: 14 });
|
|
assert.equal(aiTierNeeded(noisy.rgba, W, H, noisy.hole, (Math.PI * 12 * 12) / FRAME_AREA), false);
|
|
|
|
// ...and the same picture without the noise reads the same way, so noise is not what decided it.
|
|
const clean = frame(grain, { r: 12 });
|
|
assert.equal(aiTierNeeded(clean.rgba, W, H, clean.hole, (Math.PI * 12 * 12) / FRAME_AREA), false);
|
|
|
|
// 5. The mask handed to the model: grown by exactly the radius, and no further. The growth is a
|
|
// box, so a pixel may sit up to r diagonally away — but never outside the r-wide slab.
|
|
const grown = dilateMask(speck.hole, W, H, 3);
|
|
let grew = 0, shrank = 0, tooFar = 0;
|
|
for (let y = 0; y < H; y++) {
|
|
for (let x = 0; x < W; x++) {
|
|
const i = y * W + x;
|
|
if (!speck.hole[i] && grown[i]) {
|
|
grew++;
|
|
const d = Math.hypot(x - 256, y - 256);
|
|
if (d > 12 + 3 * Math.SQRT2 + 0.5) tooFar++;
|
|
}
|
|
if (speck.hole[i] && !grown[i]) shrank++;
|
|
}
|
|
}
|
|
assert.equal(shrank, 0, 'a grown mask still covers the stroke');
|
|
assert.ok(grew > 200, `the mask must actually grow (a 3px ring on a 12px disc is ~250 pixels, got ${grew})`);
|
|
assert.equal(tooFar, 0, 'and not by more than the radius asked for');
|
|
|
|
// 6. Grain handed back. The patch is what a model returns — a smooth surface — and the ground
|
|
// under the same mask is the photo's own noise. After the call the patch's high-pass may no
|
|
// longer be a fraction of the ground's, and the pixels outside the mask are untouched.
|
|
const noisyGround = frame(grain, { r: 26, noise: 16 });
|
|
const smoothPatch = new Uint8Array(4 * W * H);
|
|
for (let i = 0; i < W * H; i++) {
|
|
const p = i * 4;
|
|
smoothPatch[p] = smoothPatch[p + 1] = smoothPatch[p + 2] = 128;
|
|
smoothPatch[p + 3] = 255;
|
|
}
|
|
const before = smoothPatch.slice();
|
|
const hp = (data, mask, inside, stride) => {
|
|
let sum = 0, sum2 = 0, n = 0;
|
|
for (let y = 1; y < H - 1; y += stride) {
|
|
for (let x = 1; x < W - 1; x += stride) {
|
|
const i = y * W + x;
|
|
if (inside ? !mask[i] : mask[i]) continue;
|
|
const m = (data[i - W - 1] + data[i - W] + data[i - W + 1] + data[i - 1] + data[i] + data[i + 1] + data[i + W - 1] + data[i + W] + data[i + W + 1]) / 9;
|
|
const d = data[i] - m;
|
|
sum += d;
|
|
sum2 += d * d;
|
|
n++;
|
|
}
|
|
}
|
|
return Math.sqrt(Math.max(0, sum2 / n - (sum / n) ** 2));
|
|
};
|
|
const groundPlane = new Uint8Array(W * H);
|
|
for (let i = 0; i < W * H; i++) groundPlane[i] = noisyGround.rgba[i * 4];
|
|
// matchGrain reads the source the way the server is fed it: three planes, no alpha.
|
|
const src3 = new Uint8Array(3 * W * H);
|
|
for (let i = 0; i < W * H; i++) {
|
|
src3[i] = noisyGround.rgba[i * 4];
|
|
src3[W * H + i] = noisyGround.rgba[i * 4 + 1];
|
|
src3[2 * W * H + i] = noisyGround.rgba[i * 4 + 2];
|
|
}
|
|
const patchPlane = new Uint8Array(W * H);
|
|
for (let i = 0; i < W * H; i++) patchPlane[i] = smoothPatch[i * 4];
|
|
const groundSigma = hp(groundPlane, noisyGround.hole, false, 1);
|
|
const smoothSigma = hp(patchPlane, noisyGround.hole, true, 2);
|
|
const added = matchGrain(smoothPatch, src3, noisyGround.hole, W, H);
|
|
for (let i = 0; i < W * H; i++) patchPlane[i] = smoothPatch[i * 4];
|
|
const nowSigma = hp(patchPlane, noisyGround.hole, true, 2);
|
|
let touchedOutside = 0;
|
|
for (let i = 0; i < W * H; i++) {
|
|
if (noisyGround.hole[i]) continue;
|
|
const p = i * 4;
|
|
if (smoothPatch[p] !== before[p]) touchedOutside++;
|
|
}
|
|
console.log(
|
|
`ground hp sigma ${groundSigma.toFixed(2)}, model patch ${smoothSigma.toFixed(2)}, ` +
|
|
`after grain ${nowSigma.toFixed(2)} (added ${added.toFixed(2)})`
|
|
);
|
|
assert.ok(groundSigma > 3, 'the synthetic ground has to be noisy for this to mean anything');
|
|
assert.ok(nowSigma > groundSigma * 0.5, 'the model patch must end up carrying the ground grain');
|
|
assert.ok(nowSigma < groundSigma * 2.2, 'and not more of it than the ground has');
|
|
assert.equal(touchedOutside, 0, 'grain goes under the mask only');
|
|
|
|
// 7. ...and a patch that already has the ground's grain is not given a second helping.
|
|
const already = noisyGround.rgba.slice();
|
|
const addedAgain = matchGrain(already, src3, noisyGround.hole, W, H);
|
|
console.log(`already-grained patch, added ${addedAgain.toFixed(2)}`);
|
|
assert.ok(addedAgain < 1.5, `a grained patch is left alone (added ${addedAgain.toFixed(2)})`);
|
|
|
|
// 8. The fourth routing rule, the one only the engine knows: a mask no translation of the photo
|
|
// can clear. There is no patch to copy, so the mask is the model's whatever its area — and
|
|
// what the fast engine lays down instead has to be the smooth field of its own rim, with no
|
|
// direction in it. That is what the fan (radial rays converging on the centre) and the
|
|
// repeated stamp of a walked translation both are, and neither may come back from here.
|
|
const wide = frame(flat, { r: 250 }); // wider than half the crop in every direction: no shift clears it
|
|
// The mark the brush left behind, so "rebuilt" can be told from "left standing": everything under
|
|
// the brush is painted dark before the fill is asked for.
|
|
const marked = frame(flat, { r: 250 });
|
|
for (let i = 0; i < W * H; i++) {
|
|
if (!marked.hole[i]) continue;
|
|
const p = i * 4;
|
|
marked.rgba[p] = marked.rgba[p + 1] = marked.rgba[p + 2] = 30;
|
|
}
|
|
const wideFill = fastLocalInpaint(marked.rgba, W, H, marked.hole);
|
|
assert.equal(localFillWasFlat(), true, 'a hole that no translation clears must report no donor');
|
|
assert.equal(localFillWasFlat(), true, 'the flag has to survive the call the router reads it after');
|
|
let leftStanding = 0, farFromRim = 0;
|
|
for (let i = 0; i < W * H; i++) {
|
|
if (!marked.hole[i]) continue;
|
|
if (wideFill[i * 4] <= 31) leftStanding++;
|
|
if (Math.abs(wideFill[i * 4] - 128) > 60) farFromRim++;
|
|
}
|
|
assert.equal(leftStanding, 0, `${leftStanding} pixels under the brush still carry the mark`);
|
|
assert.equal(farFromRim, 0, `${farFromRim} filled pixels land away from the ground's own tone`);
|
|
|
|
// ...and the fill itself, on a ground with structure in it. What is asked of it is not accuracy
|
|
// — a hole this size is the model's, and SOR on a 500px disc converges short of the rim's own
|
|
// gradient, which is a smooth flattening and not an artifact — but that it invents no structure:
|
|
// it stays monotone across a ramp (no ridges, no crumpled planes) and it carries no fan.
|
|
const ramp = frame((x) => 30 + x * 0.4, { r: 250 });
|
|
const rampFill = fastLocalInpaint(ramp.rgba, W, H, ramp.hole);
|
|
let worstRamp = 0, drops = 0, prev = null;
|
|
for (let y = 0; y < H; y++) {
|
|
for (let x = 0; x < W; x++) {
|
|
if (!ramp.hole[y * W + x]) continue;
|
|
const got = rampFill[(y * W + x) * 4];
|
|
worstRamp = Math.max(worstRamp, Math.abs(got - luma(30 + x * 0.4)));
|
|
if (prev !== null && got < prev - 1) drops++;
|
|
prev = got;
|
|
}
|
|
prev = null;
|
|
}
|
|
console.log(`wide ramp fill: worst ${worstRamp.toFixed(1)} levels off, ${drops} non-monotone pixels`);
|
|
assert.equal(drops, 0, `the flat fill must not ripple across a ramp (${drops} pixels go backwards)`);
|
|
assert.ok(worstRamp < 40, `the residual of an unconverged wide solve stays a flattening (${worstRamp.toFixed(1)} levels)`);
|
|
|
|
const striped = frame((x, y) => 128 + Math.sin(y * 0.6) * 24, { r: 250 });
|
|
const stripedFill = fastLocalInpaint(striped.rgba, W, H, striped.hole);
|
|
const bins = new Array(72).fill(0), cnt = new Array(72).fill(0);
|
|
for (let y = 0; y < H; y++) {
|
|
for (let x = 0; x < W; x++) {
|
|
const i = y * W + x;
|
|
if (!striped.hole[i]) continue;
|
|
const dx = x - 256, dy = y - 256;
|
|
const d = Math.hypot(dx, dy) / 250;
|
|
if (d < 0.25 || d > 0.8) continue;
|
|
const a = Math.round(((Math.atan2(dy, dx) + Math.PI) / (2 * Math.PI)) * 72) % 72;
|
|
bins[a] += stripedFill[i * 4];
|
|
cnt[a]++;
|
|
}
|
|
}
|
|
const prof = bins.filter((_, i) => cnt[i]).map((v, i) => v / cnt[i]);
|
|
const pm = prof.reduce((a, b) => a + b, 0) / prof.length;
|
|
const fan = (100 * Math.sqrt(prof.reduce((a, b) => a + (b - pm) ** 2, 0) / prof.length)) / pm;
|
|
console.log(`wide striped fill: angular spread ${fan.toFixed(1)}% of the mean (the fan measure)`);
|
|
assert.ok(fan < 3, `a wide hole comes back without fan sectors (${fan.toFixed(1)}% — the old fill ran 9-39%)`);
|
|
|
|
console.log('PASS heal-route-check');
|