3dtours
bbc64118a5
- backend: preserve aspect ratio with edge-replicated square padding into 512x512 LaMa model - backend: fix inverted mask threshold (>0.85*plane) and prune CUDA/TensorRT binaries for slim image - frontend: enable server AI by default and route masks >3.5% or edge-crossing to LaMa Tier 2 - frontend: enforce 60%-80% context padding (min 64px) aligned to multiples of 8 - frontend: add 3-5px binary mask dilation and high-frequency sensor grain matching - frontend: anchor deletion badge to topmost point of brush stroke - build & export: updated docker containers and recipescam-images.tar.gz (357MB)
93 lines
4.2 KiB
JavaScript
93 lines
4.2 KiB
JavaScript
// The server's half of HEAL: lama_fp32.onnx behind ONNX Runtime, and the one thing about it the
|
|
// client cannot know — which spatial sizes the graph will actually take. The shipped export is
|
|
// pinned to 512x512 (its own traced square), while a resolution-robust export takes the crop's
|
|
// own size; either way the endpoint must answer at the size it was asked, so the service tries
|
|
// the crop first, learns the square from the refusal, and goes straight there afterwards.
|
|
//
|
|
// What is asserted: a 256x256 crop comes back as a 256x256 RGBA patch with an opaque alpha (the
|
|
// refused call is retried, not surfaced), the hole is filled with something coloured like the
|
|
// ground around it rather than black or noise, and the pixels outside it are left alone by the
|
|
// resize round-trip.
|
|
//
|
|
// node test/inpaint-onnx.mjs (from docker/backend/)
|
|
import assert from 'node:assert/strict';
|
|
import { existsSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
|
|
import { dirname, join } from 'node:path';
|
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
import ts from 'typescript';
|
|
|
|
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
|
|
const MODEL = join(ROOT, 'models', 'lama_fp32.onnx');
|
|
if (!existsSync(MODEL)) {
|
|
console.log(`SKIP no model at ${MODEL}`);
|
|
process.exit(0);
|
|
}
|
|
|
|
// Transpiled next to node_modules, not in /tmp: the service imports onnxruntime-node.
|
|
const dir = mkdtempSync(join(ROOT, 'node_modules', '.inpaint-onnx-'));
|
|
writeFileSync(
|
|
join(dir, 'inpaintService.mjs'),
|
|
ts.transpileModule(readFileSync(join(ROOT, 'src', 'inpaintService.ts'), 'utf8'), {
|
|
compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2022 },
|
|
}).outputText
|
|
);
|
|
process.env.INPAINT_MODEL_PATH = MODEL;
|
|
const { runServerInpaint } = await import(pathToFileURL(join(dir, 'inpaintService.mjs')).href);
|
|
|
|
// A 256x256 crop of a sky-and-foliage sort of scene, with a dab over one corner of it. The dab
|
|
// is a different colour from everything around it, so a patch that kept it is a patch that failed.
|
|
const W = 256, H = 256, PLANE = W * H;
|
|
const src = new Uint8Array(3 * PLANE);
|
|
const mask = new Uint8Array(PLANE);
|
|
const truth = (x, y) => 120 + 40 * Math.sin(x * 0.03) + 25 * Math.cos(y * 0.05);
|
|
for (let y = 0; y < H; y++) {
|
|
for (let x = 0; x < W; x++) {
|
|
const i = y * W + x;
|
|
const v = truth(x, y) + 8 * Math.sin(x * 0.7 + y * 0.9);
|
|
src[i] = Math.max(0, Math.min(255, v));
|
|
src[PLANE + i] = Math.max(0, Math.min(255, v * 0.95 + 6));
|
|
src[2 * PLANE + i] = Math.max(0, Math.min(255, v * 0.8 + 20));
|
|
if (x > 90 && x < 130 && y > 90 && y < 130) {
|
|
mask[i] = 255;
|
|
src[i] = 250;
|
|
src[PLANE + i] = 10;
|
|
src[2 * PLANE + i] = 250; // magenta mark, nothing like the ground
|
|
}
|
|
}
|
|
}
|
|
|
|
const t0 = Date.now();
|
|
const patch = await runServerInpaint(src, mask, W, H);
|
|
const ms = Date.now() - t0;
|
|
assert.ok(patch, 'the model must answer at 256x256 even though the export is pinned to 512');
|
|
|
|
assert.equal(patch.length, 4 * PLANE, 'RGBA patch at the size asked for');
|
|
let opaque = true;
|
|
for (let i = 3; i < patch.length; i += 4) if (patch[i] !== 255) opaque = false;
|
|
assert.ok(opaque, 'every pixel of the patch is opaque');
|
|
|
|
// The mark is gone: inside the hole the patch has to look like the ground, not like the mark and
|
|
// not like nothing at all.
|
|
let holeErr = 0, holeN = 0, outsideErr = 0, outsideN = 0;
|
|
for (let y = 0; y < H; y++) {
|
|
for (let x = 0; x < W; x++) {
|
|
const i = y * W + x, p = i * 4;
|
|
const want = truth(x, y);
|
|
const got = 0.299 * patch[p] + 0.587 * patch[p + 1] + 0.114 * patch[p + 2];
|
|
if (mask[i]) {
|
|
holeErr += Math.abs(got - want);
|
|
holeN++;
|
|
} else {
|
|
const srcLuma = 0.299 * src[i] + 0.587 * src[PLANE + i] + 0.114 * src[2 * PLANE + i];
|
|
outsideErr += Math.abs(got - srcLuma);
|
|
outsideN++;
|
|
}
|
|
}
|
|
}
|
|
const holeMae = holeErr / Math.max(1, holeN);
|
|
const outsideMae = outsideErr / Math.max(1, outsideN);
|
|
console.log(`patch ${W}x${H} in ${ms}ms, hole MAE ${holeMae.toFixed(2)}, outside MAE ${outsideMae.toFixed(2)}`);
|
|
assert.ok(holeMae < 25, `hole MAE ${holeMae.toFixed(2)} — the mark must be gone, not kept`);
|
|
assert.ok(outsideMae < 6, `outside MAE ${outsideMae.toFixed(2)} — clean pixels must survive the round trip`);
|
|
console.log('PASS inpaint-onnx');
|