3dtours/RecipesCam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(heal): upgrade inpaint architecture to hybrid 2-tier LaMa ONNX and fix starburst artifacts

Browse Source 
- backend: preserve aspect ratio with edge-replicated square padding into 512x512 LaMa model
- backend: fix inverted mask threshold (>0.85*plane) and prune CUDA/TensorRT binaries for slim image
- frontend: enable server AI by default and route masks >3.5% or edge-crossing to LaMa Tier 2
- frontend: enforce 60%-80% context padding (min 64px) aligned to multiples of 8
- frontend: add 3-5px binary mask dilation and high-frequency sensor grain matching
- frontend: anchor deletion badge to topmost point of brush stroke
- build & export: updated docker containers and recipescam-images.tar.gz (357MB)
This commit is contained in:
3dtours
2026-10-17 11:01:00 +07:00
parent 2761d4f5a3
commit bbc64118a5
14 changed files with 1242 additions and 415 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/backend/Dockerfile
+8
View File
@@ -7,6 +7,14 @@ RUN npm ci
COPY tsconfig.json ./
COPY src ./src
RUN npm run build && npm prune --omit=dev
# onnxruntime-node ships every platform's runtime, and 260MB of it is the CUDA /
# TensorRT providers this container (CPU inference, no GPU) never loads. The
# bindings for the other two operating systems go the same way; both Linux
# architectures stay so the image can still be built for an ARM host.
RUN rm -rf node_modules/onnxruntime-node/bin/napi-v6/darwin \
node_modules/onnxruntime-node/bin/napi-v6/win32 \
node_modules/onnxruntime-node/bin/napi-v6/linux/x64/libonnxruntime_providers_cuda.so \
node_modules/onnxruntime-node/bin/napi-v6/linux/x64/libonnxruntime_providers_tensorrt.so
FROM node:22-slim
ENV NODE_ENV=production \
docker/backend/package.json
+2 -1
View File
@@ -9,7 +9,8 @@
"start": "node dist/server.js",
"dev": "tsx watch src/server.ts",
"test": "node test/security.mjs",
"test:immich": "node test/immich.mjs"
"test:immich": "node test/immich.mjs",
"test:inpaint": "node test/inpaint-onnx.mjs"
},
"dependencies": {
"better-sqlite3": "^12.11.1",
docker/backend/src/inpaintService.ts
+101 -20
View File
@@ -1,8 +1,13 @@
import * as ort from 'onnxruntime-node';
import { existsSync } from 'node:fs';
import { cpus } from 'node:os';
const MODEL_PATH = process.env.INPAINT_MODEL_PATH || '/models/lama_fp32.onnx';
// What the graph will take spatially, learned on the first call: a resolution-robust export runs
// at the crop's own size, a pinned one reports the square it was traced at and we remember it.
let fixedSide: number | null = null;
let sessionPromise: Promise<ort.InferenceSession | null> | null = null;
export async function getInpaintSession(): Promise<ort.InferenceSession | null> {
@@ -13,8 +18,6 @@ export async function getInpaintSession(): Promise<ort.InferenceSession | null>
'/app/models/lama_fp32.onnx',
'/models/lama_fp32.onnx',
'./models/lama_fp32.onnx',
'/home/locpham/RecipesCam/docker/frontend/public/models/lama_fp32.onnx',
'../frontend/public/models/lama_fp32.onnx',
];
let foundPath: string | null = null;
for (const p of candidates) {
@@ -33,6 +36,9 @@ export async function getInpaintSession(): Promise<ort.InferenceSession | null>
const session = await ort.InferenceSession.create(foundPath, {
executionProviders: ['cpu'],
graphOptimizationLevel: 'all',
// Measured on the dev box's 4-core i3: 4 threads 3.0s, default (8) 3.8s, 2 threads 4.0s.
// Half the cores is the knee, and it leaves the rest of the API its share on a NAS.
intraOpNumThreads: Math.max(1, Math.min(4, Math.floor(cpus().length / 2))),
});
console.log('[inpaintService] Model loaded in ' + (Date.now() - t0) + 'ms');
console.log('[inpaintService] Inputs:', session.inputNames, 'Outputs:', session.outputNames);
@@ -106,38 +112,113 @@ export async function runServerInpaint(
rawRgb[i] = srcBytes[i] / 255.0;
}
// Any non-zero byte is the hole: the model's convention is a binary white-on-black mask, and
// the caller's white may arrive as 255, 1 or anything between. A mask that reads as all-zero
// here is not a repair at all — LaMa answers with the image it was handed — so the byte that
// says "hole" is worth accepting in every spelling.
// Inversion check: only flip if > 85% of the patch is marked as hole (suggesting inverted alpha mask)
const rawMask = new Float32Array(plane);
let holes = 0;
for (let i = 0; i < plane; i++) {
rawMask[i] = maskBytes[i] > 128 ? 1.0 : 0.0;
if (maskBytes[i] !== 0) {
rawMask[i] = 1.0;
holes++;
}
}
// Bilinear resize to 512x512 for LaMa
const inImage512 = resizeBilinear(rawRgb, w, h, 512, 512, 3);
const inMask512 = resizeBilinear(rawMask, w, h, 512, 512, 1);
for (let i = 0; i < 512 * 512; i++) {
inMask512[i] = inMask512[i] > 0.15 ? 1.0 : 0.0;
if (holes > 0.85 * plane) {
console.warn(`[inpaintService] mask is ${Math.round((100 * holes) / plane)}% of the crop: read as inverted, flipping`);
for (let i = 0; i < plane; i++) rawMask[i] = rawMask[i] > 0 ? 0 : 1;
}
console.log(`[inpaintService] mask ${w}x${h}: ${holes} hole px of ${plane} (${((100 * holes) / plane).toFixed(1)}%)`);
const imageInputName = session.inputNames.includes('image') ? 'image' : session.inputNames[0];
const maskInputName = session.inputNames.includes('mask') ? 'mask' : session.inputNames[1];
const out = await session.run({
[imageInputName]: new ort.Tensor('float32', inImage512, [1, 3, 512, 512]),
[maskInputName]: new ort.Tensor('float32', inMask512, [1, 1, 512, 512]),
});
// Uniform aspect-ratio preserving resize and padding to fit the side the graph accepts (e.g. 512).
// Preserving aspect ratio prevents anamorphic squashing/stretching that causes radial starburst
// and Fourier spectral leakage artifacts.
const runAt = async (side: number): Promise<Float32Array> => {
const scale = Math.min(side / w, side / h);
const scaledW = Math.max(1, Math.min(side, Math.round(w * scale)));
const scaledH = Math.max(1, Math.min(side, Math.round(h * scale)));
const outTensor = out[session.outputNames[0]];
const out512 = outTensor.data as Float32Array;
const resizedImg = resizeBilinear(rawRgb, w, h, scaledW, scaledH, 3);
const resizedMask = resizeBilinear(rawMask, w, h, scaledW, scaledH, 1);
// Resize output back to w x h
const outSw = resizeBilinear(out512, 512, 512, w, h, 3);
// Pad into side x side buffer with edge replication for image and 0 for mask
const inImage = new Float32Array(3 * side * side);
const inMask = new Float32Array(side * side);
for (let y = 0; y < side; y++) {
const sy = Math.min(scaledH - 1, y);
for (let x = 0; x < side; x++) {
const sx = Math.min(scaledW - 1, x);
const dstIdx = y * side + x;
const srcIdx = sy * scaledW + sx;
inImage[dstIdx] = resizedImg[srcIdx];
inImage[side * side + dstIdx] = resizedImg[scaledW * scaledH + srcIdx];
inImage[2 * side * side + dstIdx] = resizedImg[2 * scaledW * scaledH + srcIdx];
if (y < scaledH && x < scaledW) {
inMask[dstIdx] = resizedMask[srcIdx] > 0.15 ? 1.0 : 0.0;
}
}
}
const out = await session.run({
[imageInputName]: new ort.Tensor('float32', inImage, [1, 3, side, side]),
[maskInputName]: new ort.Tensor('float32', inMask, [1, 1, side, side]),
});
const outTensor = out[session.outputNames[0]];
const outData = outTensor.data as Float32Array;
// Extract the scaledW x scaledH subregion
const croppedOut = new Float32Array(3 * scaledW * scaledH);
for (let y = 0; y < scaledH; y++) {
for (let x = 0; x < scaledW; x++) {
const srcIdx = y * side + x;
const dstIdx = y * scaledW + x;
croppedOut[dstIdx] = outData[srcIdx];
croppedOut[scaledW * scaledH + dstIdx] = outData[side * side + srcIdx];
croppedOut[2 * scaledW * scaledH + dstIdx] = outData[2 * side * side + srcIdx];
}
}
// Bilinear resize back to crop w x h
return resizeBilinear(croppedOut, scaledW, scaledH, w, h, 3);
};
let outSw: Float32Array;
if (fixedSide) {
outSw = await runAt(fixedSide);
} else {
const side = Math.max(64, Math.min(768, Math.ceil(Math.max(w, h) / 8) * 8));
try {
outSw = await runAt(side);
} catch (err) {
const expected = /Expected: (\d+)/.exec(String((err as Error)?.message || err));
if (!expected) throw err;
fixedSide = Number(expected[1]);
outSw = await runAt(fixedSide);
}
}
// Adaptive scale: detect if model output is [0..1] vs [0..255]
let maxVal = -Infinity;
const testLen = Math.min(500, outSw.length);
for (let i = 0; i < testLen; i++) {
if (outSw[i] > maxVal) maxVal = outSw[i];
}
const mult = maxVal <= 2.5 && maxVal > 0 ? 255.0 : 1.0;
const patch = new Uint8Array(4 * plane);
const plane2 = 2 * plane;
for (let i = 0; i < plane; i++) {
const r = Math.round(outSw[i]);
const g = Math.round(outSw[plane + i]);
const b = Math.round(outSw[plane2 + i]);
const r = Math.round(outSw[i] * mult);
const g = Math.round(outSw[plane + i] * mult);
const b = Math.round(outSw[plane2 + i] * mult);
patch[i * 4] = r < 0 ? 0 : r > 255 ? 255 : r;
patch[i * 4 + 1] = g < 0 ? 0 : g > 255 ? 255 : g;
docker/backend/test/inpaint-onnx.mjs
+92
View File
@@ -0,0 +1,92 @@
// The server's half of HEAL: lama_fp32.onnx behind ONNX Runtime, and the one thing about it the
// client cannot know — which spatial sizes the graph will actually take. The shipped export is
// pinned to 512x512 (its own traced square), while a resolution-robust export takes the crop's
// own size; either way the endpoint must answer at the size it was asked, so the service tries
// the crop first, learns the square from the refusal, and goes straight there afterwards.
//
// What is asserted: a 256x256 crop comes back as a 256x256 RGBA patch with an opaque alpha (the
// refused call is retried, not surfaced), the hole is filled with something coloured like the
// ground around it rather than black or noise, and the pixels outside it are left alone by the
// resize round-trip.
//
// node test/inpaint-onnx.mjs (from docker/backend/)
import assert from 'node:assert/strict';
import { existsSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import ts from 'typescript';
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
const MODEL = join(ROOT, 'models', 'lama_fp32.onnx');
if (!existsSync(MODEL)) {
console.log(`SKIP no model at ${MODEL}`);
process.exit(0);
}
// Transpiled next to node_modules, not in /tmp: the service imports onnxruntime-node.
const dir = mkdtempSync(join(ROOT, 'node_modules', '.inpaint-onnx-'));
writeFileSync(
join(dir, 'inpaintService.mjs'),
ts.transpileModule(readFileSync(join(ROOT, 'src', 'inpaintService.ts'), 'utf8'), {
compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2022 },
}).outputText
);
process.env.INPAINT_MODEL_PATH = MODEL;
const { runServerInpaint } = await import(pathToFileURL(join(dir, 'inpaintService.mjs')).href);
// A 256x256 crop of a sky-and-foliage sort of scene, with a dab over one corner of it. The dab
// is a different colour from everything around it, so a patch that kept it is a patch that failed.
const W = 256, H = 256, PLANE = W * H;
const src = new Uint8Array(3 * PLANE);
const mask = new Uint8Array(PLANE);
const truth = (x, y) => 120 + 40 * Math.sin(x * 0.03) + 25 * Math.cos(y * 0.05);
for (let y = 0; y < H; y++) {
for (let x = 0; x < W; x++) {
const i = y * W + x;
const v = truth(x, y) + 8 * Math.sin(x * 0.7 + y * 0.9);
src[i] = Math.max(0, Math.min(255, v));
src[PLANE + i] = Math.max(0, Math.min(255, v * 0.95 + 6));
src[2 * PLANE + i] = Math.max(0, Math.min(255, v * 0.8 + 20));
if (x > 90 && x < 130 && y > 90 && y < 130) {
mask[i] = 255;
src[i] = 250;
src[PLANE + i] = 10;
src[2 * PLANE + i] = 250; // magenta mark, nothing like the ground
}
}
}
const t0 = Date.now();
const patch = await runServerInpaint(src, mask, W, H);
const ms = Date.now() - t0;
assert.ok(patch, 'the model must answer at 256x256 even though the export is pinned to 512');
assert.equal(patch.length, 4 * PLANE, 'RGBA patch at the size asked for');
let opaque = true;
for (let i = 3; i < patch.length; i += 4) if (patch[i] !== 255) opaque = false;
assert.ok(opaque, 'every pixel of the patch is opaque');
// The mark is gone: inside the hole the patch has to look like the ground, not like the mark and
// not like nothing at all.
let holeErr = 0, holeN = 0, outsideErr = 0, outsideN = 0;
for (let y = 0; y < H; y++) {
for (let x = 0; x < W; x++) {
const i = y * W + x, p = i * 4;
const want = truth(x, y);
const got = 0.299 * patch[p] + 0.587 * patch[p + 1] + 0.114 * patch[p + 2];
if (mask[i]) {
holeErr += Math.abs(got - want);
holeN++;
} else {
const srcLuma = 0.299 * src[i] + 0.587 * src[PLANE + i] + 0.114 * src[2 * PLANE + i];
outsideErr += Math.abs(got - srcLuma);
outsideN++;
}
}
}
const holeMae = holeErr / Math.max(1, holeN);
const outsideMae = outsideErr / Math.max(1, outsideN);
console.log(`patch ${W}x${H} in ${ms}ms, hole MAE ${holeMae.toFixed(2)}, outside MAE ${outsideMae.toFixed(2)}`);
assert.ok(holeMae < 25, `hole MAE ${holeMae.toFixed(2)} — the mark must be gone, not kept`);
assert.ok(outsideMae < 6, `outside MAE ${outsideMae.toFixed(2)} — clean pixels must survive the round trip`);
console.log('PASS inpaint-onnx');