Files
RecipesCam/docker/frontend/scripts/heal-route-check.mjs
T
3dtours bbc64118a5 fix(heal): upgrade inpaint architecture to hybrid 2-tier LaMa ONNX and fix starburst artifacts
- backend: preserve aspect ratio with edge-replicated square padding into 512x512 LaMa model
- backend: fix inverted mask threshold (>0.85*plane) and prune CUDA/TensorRT binaries for slim image
- frontend: enable server AI by default and route masks >3.5% or edge-crossing to LaMa Tier 2
- frontend: enforce 60%-80% context padding (min 64px) aligned to multiples of 8
- frontend: add 3-5px binary mask dilation and high-frequency sensor grain matching
- frontend: anchor deletion badge to topmost point of brush stroke
- build & export: updated docker containers and recipescam-images.tar.gz (357MB)
2026-10-17 11:01:00 +07:00

254 lines
13 KiB
JavaScript

// The hybrid's three client-side rules, measured on synthetic frames whose answer is known:
//
// * the router — which of the two engines a mask goes to (a speck of dust on open ground is the
// fast engine's job and costs nothing; an object, or a mark that sits across a real edge,
// goes to the model),
// * the mask the model is given — grown a few pixels over the stroke so the object's shadow
// goes with it, and only over the stroke, because the brush is not the wound,
// * the grain the model leaves behind — a model patch is a clean surface, and a clean surface
// in a photograph of sensor noise reads as a smudge, so it is given back the ground's own
// grain and nothing where the ground has none.
//
// node scripts/heal-route-check.mjs
import assert from 'node:assert/strict';
import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { pathToFileURL } from 'node:url';
import ts from 'typescript';
const SRC = new URL('../src/engine/inpaint.ts', import.meta.url);
const dir = mkdtempSync(join(tmpdir(), 'heal-route-'));
writeFileSync(
join(dir, 'inpaint.mjs'),
ts.transpileModule(readFileSync(SRC, 'utf8'), {
compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2022 },
}).outputText
);
const { aiTierNeeded, dilateMask, matchGrain, AI_MAX_MASK_AREA, fastLocalInpaint, localFillWasFlat } = await import(
pathToFileURL(join(dir, 'inpaint.mjs')).href
);
const W = 512, H = 512;
const luma = (v) => Math.max(0, Math.min(255, v));
// A frame builder: `at(x, y)` says what the ground looks like, and the hole is a disc on it.
function frame(at, { cx = 256, cy = 256, r = 12, noise = 0, seed = 7 } = {}) {
const rgba = new Uint8Array(W * H * 4);
const hole = new Uint8Array(W * H);
let s = seed;
const rnd = () => ((s = (s * 1103515245 + 12345) & 0x7fffffff) / 0x7fffffff) - 0.5;
for (let y = 0; y < H; y++) {
for (let x = 0; x < W; x++) {
const i = y * W + x, p = i * 4;
const v = luma(at(x, y) + (noise ? rnd() * noise : 0));
rgba[p] = rgba[p + 1] = rgba[p + 2] = v;
rgba[p + 3] = 255;
if (r > 0 && (x - cx) ** 2 + (y - cy) ** 2 <= r * r) hole[i] = 1;
}
}
return { rgba, hole };
}
const flat = () => 128;
const steps = (x) => (x < 128 || x > 380 ? 80 : 176);
const grain = () => 128;
const FRAME_AREA = W * H; // this frame is the whole picture for the routing tests
// 1. A speck of dust on open ground: the fast engine, no server.
const speck = frame(flat, { r: 12 });
assert.equal(aiTierNeeded(speck.rgba, W, H, speck.hole, (Math.PI * 12 * 12) / FRAME_AREA), false);
// 2. Something the size of an object: the model. On this 512x512 frame a 60px radius is 4.3% of
// the area, which is the same order as a 5%-of-width brush on a 3:2 frame.
const object = frame(flat, { r: 60 });
assert.equal(aiTierNeeded(object.rgba, W, H, object.hole, (Math.PI * 60 * 60) / FRAME_AREA), true);
assert.ok(
(Math.PI * 60 * 60) / FRAME_AREA > AI_MAX_MASK_AREA,
'the object case must be over the 3.5% the chart draws'
);
// 2b. The brush at its widest on open ground is still a speck by that measure: the fast engine.
const wideDab = frame(flat, { r: 26 });
assert.equal(aiTierNeeded(wideDab.rgba, W, H, wideDab.hole, (Math.PI * 26 * 26) / FRAME_AREA), false);
// 2c. With no frame to judge the area against, the window is what is left — and a real window is
// a tight one around its hole, so a mark that fills an eighth of it goes to the model.
const filled = frame(flat, { r: 100 });
assert.equal(aiTierNeeded(filled.rgba, W, H, filled.hole), true);
// 3. A speck, but one sitting across a real edge: the model, however small it is. The ground here
// is quiet (rim 1.88 of 255 either side of the line) so the area test would have kept it home;
// what sends it out is that the rim carries six times the window's own structure.
const onEdge = frame(steps, { cx: 128, cy: 256, r: 12 });
assert.equal(aiTierNeeded(onEdge.rgba, W, H, onEdge.hole, (Math.PI * 12 * 12) / FRAME_AREA), true);
// 3b. Busy ground — foliage, water ripples, a crowd — is where a copied patch shows up as a slab
// of somewhere else, so a speck there goes to the model too, even with no edge in sight.
const foliage = frame((x, y) => 120 + 50 * Math.abs(Math.sin(x * 0.21 + y * 0.13)) + 30 * Math.sin(x * 0.7 + y * 1.3), { r: 26 });
assert.equal(aiTierNeeded(foliage.rgba, W, H, foliage.hole, (Math.PI * 26 * 26) / FRAME_AREA), true);
// 3c. ...but ripples are not that busy, and the engine does answer them: a speck on water stays.
const water = frame((x, y) => 128 + 18 * Math.sin(x * 0.35 + y * 0.5) + 10 * Math.sin(x * 0.9 - y * 0.7), { r: 26 });
assert.equal(aiTierNeeded(water.rgba, W, H, water.hole, (Math.PI * 26 * 26) / FRAME_AREA), false);
// 4. A speck on ground that is nothing but sensor grain: the fast engine. Grain is not structure.
const noisy = frame(grain, { r: 12, noise: 14 });
assert.equal(aiTierNeeded(noisy.rgba, W, H, noisy.hole, (Math.PI * 12 * 12) / FRAME_AREA), false);
// ...and the same picture without the noise reads the same way, so noise is not what decided it.
const clean = frame(grain, { r: 12 });
assert.equal(aiTierNeeded(clean.rgba, W, H, clean.hole, (Math.PI * 12 * 12) / FRAME_AREA), false);
// 5. The mask handed to the model: grown by exactly the radius, and no further. The growth is a
// box, so a pixel may sit up to r diagonally away — but never outside the r-wide slab.
const grown = dilateMask(speck.hole, W, H, 3);
let grew = 0, shrank = 0, tooFar = 0;
for (let y = 0; y < H; y++) {
for (let x = 0; x < W; x++) {
const i = y * W + x;
if (!speck.hole[i] && grown[i]) {
grew++;
const d = Math.hypot(x - 256, y - 256);
if (d > 12 + 3 * Math.SQRT2 + 0.5) tooFar++;
}
if (speck.hole[i] && !grown[i]) shrank++;
}
}
assert.equal(shrank, 0, 'a grown mask still covers the stroke');
assert.ok(grew > 200, `the mask must actually grow (a 3px ring on a 12px disc is ~250 pixels, got ${grew})`);
assert.equal(tooFar, 0, 'and not by more than the radius asked for');
// 6. Grain handed back. The patch is what a model returns — a smooth surface — and the ground
// under the same mask is the photo's own noise. After the call the patch's high-pass may no
// longer be a fraction of the ground's, and the pixels outside the mask are untouched.
const noisyGround = frame(grain, { r: 26, noise: 16 });
const smoothPatch = new Uint8Array(4 * W * H);
for (let i = 0; i < W * H; i++) {
const p = i * 4;
smoothPatch[p] = smoothPatch[p + 1] = smoothPatch[p + 2] = 128;
smoothPatch[p + 3] = 255;
}
const before = smoothPatch.slice();
const hp = (data, mask, inside, stride) => {
let sum = 0, sum2 = 0, n = 0;
for (let y = 1; y < H - 1; y += stride) {
for (let x = 1; x < W - 1; x += stride) {
const i = y * W + x;
if (inside ? !mask[i] : mask[i]) continue;
const m = (data[i - W - 1] + data[i - W] + data[i - W + 1] + data[i - 1] + data[i] + data[i + 1] + data[i + W - 1] + data[i + W] + data[i + W + 1]) / 9;
const d = data[i] - m;
sum += d;
sum2 += d * d;
n++;
}
}
return Math.sqrt(Math.max(0, sum2 / n - (sum / n) ** 2));
};
const groundPlane = new Uint8Array(W * H);
for (let i = 0; i < W * H; i++) groundPlane[i] = noisyGround.rgba[i * 4];
// matchGrain reads the source the way the server is fed it: three planes, no alpha.
const src3 = new Uint8Array(3 * W * H);
for (let i = 0; i < W * H; i++) {
src3[i] = noisyGround.rgba[i * 4];
src3[W * H + i] = noisyGround.rgba[i * 4 + 1];
src3[2 * W * H + i] = noisyGround.rgba[i * 4 + 2];
}
const patchPlane = new Uint8Array(W * H);
for (let i = 0; i < W * H; i++) patchPlane[i] = smoothPatch[i * 4];
const groundSigma = hp(groundPlane, noisyGround.hole, false, 1);
const smoothSigma = hp(patchPlane, noisyGround.hole, true, 2);
const added = matchGrain(smoothPatch, src3, noisyGround.hole, W, H);
for (let i = 0; i < W * H; i++) patchPlane[i] = smoothPatch[i * 4];
const nowSigma = hp(patchPlane, noisyGround.hole, true, 2);
let touchedOutside = 0;
for (let i = 0; i < W * H; i++) {
if (noisyGround.hole[i]) continue;
const p = i * 4;
if (smoothPatch[p] !== before[p]) touchedOutside++;
}
console.log(
`ground hp sigma ${groundSigma.toFixed(2)}, model patch ${smoothSigma.toFixed(2)}, ` +
`after grain ${nowSigma.toFixed(2)} (added ${added.toFixed(2)})`
);
assert.ok(groundSigma > 3, 'the synthetic ground has to be noisy for this to mean anything');
assert.ok(nowSigma > groundSigma * 0.5, 'the model patch must end up carrying the ground grain');
assert.ok(nowSigma < groundSigma * 2.2, 'and not more of it than the ground has');
assert.equal(touchedOutside, 0, 'grain goes under the mask only');
// 7. ...and a patch that already has the ground's grain is not given a second helping.
const already = noisyGround.rgba.slice();
const addedAgain = matchGrain(already, src3, noisyGround.hole, W, H);
console.log(`already-grained patch, added ${addedAgain.toFixed(2)}`);
assert.ok(addedAgain < 1.5, `a grained patch is left alone (added ${addedAgain.toFixed(2)})`);
// 8. The fourth routing rule, the one only the engine knows: a mask no translation of the photo
// can clear. There is no patch to copy, so the mask is the model's whatever its area — and
// what the fast engine lays down instead has to be the smooth field of its own rim, with no
// direction in it. That is what the fan (radial rays converging on the centre) and the
// repeated stamp of a walked translation both are, and neither may come back from here.
const wide = frame(flat, { r: 250 }); // wider than half the crop in every direction: no shift clears it
// The mark the brush left behind, so "rebuilt" can be told from "left standing": everything under
// the brush is painted dark before the fill is asked for.
const marked = frame(flat, { r: 250 });
for (let i = 0; i < W * H; i++) {
if (!marked.hole[i]) continue;
const p = i * 4;
marked.rgba[p] = marked.rgba[p + 1] = marked.rgba[p + 2] = 30;
}
const wideFill = fastLocalInpaint(marked.rgba, W, H, marked.hole);
assert.equal(localFillWasFlat(), true, 'a hole that no translation clears must report no donor');
assert.equal(localFillWasFlat(), true, 'the flag has to survive the call the router reads it after');
let leftStanding = 0, farFromRim = 0;
for (let i = 0; i < W * H; i++) {
if (!marked.hole[i]) continue;
if (wideFill[i * 4] <= 31) leftStanding++;
if (Math.abs(wideFill[i * 4] - 128) > 60) farFromRim++;
}
assert.equal(leftStanding, 0, `${leftStanding} pixels under the brush still carry the mark`);
assert.equal(farFromRim, 0, `${farFromRim} filled pixels land away from the ground's own tone`);
// ...and the fill itself, on a ground with structure in it. What is asked of it is not accuracy
// — a hole this size is the model's, and SOR on a 500px disc converges short of the rim's own
// gradient, which is a smooth flattening and not an artifact — but that it invents no structure:
// it stays monotone across a ramp (no ridges, no crumpled planes) and it carries no fan.
const ramp = frame((x) => 30 + x * 0.4, { r: 250 });
const rampFill = fastLocalInpaint(ramp.rgba, W, H, ramp.hole);
let worstRamp = 0, drops = 0, prev = null;
for (let y = 0; y < H; y++) {
for (let x = 0; x < W; x++) {
if (!ramp.hole[y * W + x]) continue;
const got = rampFill[(y * W + x) * 4];
worstRamp = Math.max(worstRamp, Math.abs(got - luma(30 + x * 0.4)));
if (prev !== null && got < prev - 1) drops++;
prev = got;
}
prev = null;
}
console.log(`wide ramp fill: worst ${worstRamp.toFixed(1)} levels off, ${drops} non-monotone pixels`);
assert.equal(drops, 0, `the flat fill must not ripple across a ramp (${drops} pixels go backwards)`);
assert.ok(worstRamp < 40, `the residual of an unconverged wide solve stays a flattening (${worstRamp.toFixed(1)} levels)`);
const striped = frame((x, y) => 128 + Math.sin(y * 0.6) * 24, { r: 250 });
const stripedFill = fastLocalInpaint(striped.rgba, W, H, striped.hole);
const bins = new Array(72).fill(0), cnt = new Array(72).fill(0);
for (let y = 0; y < H; y++) {
for (let x = 0; x < W; x++) {
const i = y * W + x;
if (!striped.hole[i]) continue;
const dx = x - 256, dy = y - 256;
const d = Math.hypot(dx, dy) / 250;
if (d < 0.25 || d > 0.8) continue;
const a = Math.round(((Math.atan2(dy, dx) + Math.PI) / (2 * Math.PI)) * 72) % 72;
bins[a] += stripedFill[i * 4];
cnt[a]++;
}
}
const prof = bins.filter((_, i) => cnt[i]).map((v, i) => v / cnt[i]);
const pm = prof.reduce((a, b) => a + b, 0) / prof.length;
const fan = (100 * Math.sqrt(prof.reduce((a, b) => a + (b - pm) ** 2, 0) / prof.length)) / pm;
console.log(`wide striped fill: angular spread ${fan.toFixed(1)}% of the mean (the fan measure)`);
assert.ok(fan < 3, `a wide hole comes back without fan sectors (${fan.toFixed(1)}% — the old fill ran 9-39%)`);
console.log('PASS heal-route-check');