3dtours/RecipesCam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(studio): redesign LIGHT tab to Lumetri style and fix mobile inpaint freeze with worker proxy and patch cache

Browse Source
This commit is contained in:
3dtours
2026-10-14 11:01:52 +07:00
parent e9cc2bb15a
commit d52d4c4281
5 changed files with 560 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/frontend/src/engine/inpaint.ts
+127 -59
View File
@@ -50,11 +50,20 @@ interface Loaded {
let loaded: Promise<Loaded> | null = null;
function isMobile(): boolean {
if (typeof navigator === 'undefined') return false;
const ua = navigator.userAgent || '';
return (
/iPad|iPhone|iPod|Android|Mobile/i.test(ua) ||
(navigator.platform === 'MacIntel' && (navigator.maxTouchPoints || 0) > 1)
);
}
function isIOS(): boolean {
if (typeof navigator === 'undefined') return false;
const ua = navigator.userAgent || '';
return (
/iPad|iPhone|iPod/.test(ua) ||
/iPad|iPhone|iPod/i.test(ua) ||
(navigator.platform === 'MacIntel' && (navigator.maxTouchPoints || 0) > 1)
);
}
@@ -123,16 +132,18 @@ function load(): Promise<Loaded> {
loaded = (async () => {
const ort = await import('onnxruntime-web');
ort.env.wasm.wasmPaths = WASM_DIR;
const ios = isIOS();
// On iOS WebKit, multi-threading or worker proxying triggers memory limits
// and WebKit crashes. Single thread direct WASM execution is rock-solid and
// fast on Apple Silicon / A-series chips.
ort.env.wasm.numThreads = ios
const mobile = isMobile();
// On mobile devices, 1 worker thread keeps memory strictly under the browser tab limit.
// On desktop, up to 4 threads if cross-origin isolated.
ort.env.wasm.numThreads = mobile
? 1
: self.crossOriginIsolated
? Math.min(4, navigator.hardwareConcurrency || 1)
: 1;
ort.env.wasm.proxy = !ios;
// ALWAYS proxy ONNX WASM execution into a Web Worker.
// Running heavy WASM inferences on the main thread locks the UI loop ("not responding")
// and triggers browser watchdog termination (crash / full reload).
ort.env.wasm.proxy = true;
ort.env.webgpu.powerPreference = 'high-performance';
const session = await openSession(ort);
return { ort, session };
@@ -159,12 +170,24 @@ export function preloadInpaint(): void {
}
}
interface CachedPatch {
key: string;
patch: Uint8Array;
}
let patchCache: CachedPatch | null = null;
let inFlightInpaint: Promise<Uint8Array | null> | null = null;
export function clearInpaintCache(): void {
patchCache = null;
}
// The page is going away, or coming back from the back/forward cache: hand the
// session back rather than leaving the browser to reap its buffers and its
// thread pool with the page. `loaded` goes with it, so a page that does come
// back loads the runtime again instead of calling into a released session — the
// files are cached, so that is a second.
export async function releaseInpaint(): Promise<void> {
patchCache = null;
const pending = loaded;
loaded = null;
if (!pending) return;
@@ -199,7 +222,7 @@ export async function inpaintCrop(
height: number,
hole: Uint8Array
): Promise<Uint8Array | null> {
const maxSide = isIOS() ? 384 : INPAINT_MAX_SIDE;
const maxSide = isMobile() ? 256 : INPAINT_MAX_SIDE;
let w = Math.max(8, Math.round(width));
let h = Math.max(8, Math.round(height));
const scale = Math.min(1, maxSide / Math.max(w, h));
@@ -210,61 +233,106 @@ export async function inpaintCrop(
w = Math.max(8, Math.floor((w * scale) / 8) * 8);
h = Math.max(8, Math.floor((h * scale) / 8) * 8);
// NCHW, 0..255 RGB — what the model was trained to read — and the mask in its
// own polarity: 255 everywhere the frame is known, 0 over the hole.
const src = new Uint8Array(3 * w * h);
const mask = new Uint8Array(w * h).fill(255);
const sx = width / w;
const sy = height / h;
const plane = w * h;
for (let y = 0; y < h; y++) {
// Nearest source pixel rather than a box filter: the crop is the frame at
// its own resolution whenever it fits, so the two agree pixel for pixel and
// this loop never runs at all in the common case.
const syy = Math.min(height - 1, Math.floor(y * sy));
for (let x = 0; x < w; x++) {
const sxx = Math.min(width - 1, Math.floor(x * sx));
const s = (syy * width + sxx) * 4;
const d = y * w + x;
src[d] = rgba[s];
src[plane + d] = rgba[s + 1];
src[2 * plane + d] = rgba[s + 2];
if (hole[syy * width + sxx]) mask[d] = 0;
// Quick signature to avoid redundant re-inferences on subsequent renders
let holeCount = 0;
let holeHash = 0;
const step = Math.max(1, Math.floor(hole.length / 64));
for (let i = 0; i < hole.length; i += step) {
if (hole[i]) {
holeCount++;
holeHash = (holeHash * 31 + i) | 0;
}
}
let pixHash = 0;
const pixStep = Math.max(4, Math.floor(rgba.length / 64) * 4);
for (let i = 0; i < rgba.length; i += pixStep) {
pixHash = (pixHash * 33 + rgba[i] + rgba[i + 1]) | 0;
}
const key = `${w}x${h}_${holeCount}_${holeHash}_${pixHash}`;
if (patchCache && patchCache.key === key) {
return patchCache.patch.slice();
}
// Single-flight lock: serialize inference executions to prevent WebAssembly memory spikes on mobile
if (inFlightInpaint) {
try {
await inFlightInpaint;
} catch {}
if (patchCache && patchCache.key === key) {
return patchCache.patch.slice();
}
}
try {
const { ort, session } = await load();
const in0 = new ort.Tensor('uint8', src, [1, 3, h, w]);
const in1 = new ort.Tensor('uint8', mask, [1, 1, h, w]);
const out = await session.run({
[session.inputNames[0]]: in0,
[session.inputNames[1]]: in1,
});
const tensor = out[session.outputNames[0]];
const oplane = w * h;
const data = tensor.data as Uint8Array;
// Back to RGBA, opaque: the crop covers the frame, which has no transparency
// of its own to carry.
const patch = new Uint8Array(4 * w * h);
for (let i = 0; i < oplane; i++) {
patch[i * 4] = data[i];
patch[i * 4 + 1] = data[oplane + i];
patch[i * 4 + 2] = data[2 * oplane + i];
patch[i * 4 + 3] = 255;
const runPromise = (async (): Promise<Uint8Array | null> => {
// NCHW, 0..255 RGB — what the model was trained to read — and the mask in its
// own polarity: 255 everywhere the frame is known, 0 over the hole.
const src = new Uint8Array(3 * w * h);
const mask = new Uint8Array(w * h).fill(255);
const sx = width / w;
const sy = height / h;
const plane = w * h;
for (let y = 0; y < h; y++) {
// Nearest source pixel rather than a box filter: the crop is the frame at
// its own resolution whenever it fits, so the two agree pixel for pixel and
// this loop never runs at all in the common case.
const syy = Math.min(height - 1, Math.floor(y * sy));
for (let x = 0; x < w; x++) {
const sxx = Math.min(width - 1, Math.floor(x * sx));
const s = (syy * width + sxx) * 4;
const d = y * w + x;
src[d] = rgba[s];
src[plane + d] = rgba[s + 1];
src[2 * plane + d] = rgba[s + 2];
if (hole[syy * width + sxx]) mask[d] = 0;
}
}
try {
in0.dispose?.();
in1.dispose?.();
tensor.dispose?.();
} catch {}
return patch;
} catch (err) {
// A model that cannot run is a soft repair, not a broken one — and this one
// line is what keeps the fallback diagnosable instead of invisible, because
// the picture that comes out of it looks like a workable repair.
console.warn('[inpaint] model unavailable, keeping the blurred fill', err);
return null;
const { ort, session } = await load();
const in0 = new ort.Tensor('uint8', src, [1, 3, h, w]);
const in1 = new ort.Tensor('uint8', mask, [1, 1, h, w]);
const out = await session.run({
[session.inputNames[0]]: in0,
[session.inputNames[1]]: in1,
});
const tensor = out[session.outputNames[0]];
const oplane = w * h;
const data = tensor.data as Uint8Array;
// Back to RGBA, opaque: the crop covers the frame, which has no transparency
// of its own to carry.
const patch = new Uint8Array(4 * w * h);
for (let i = 0; i < oplane; i++) {
patch[i * 4] = data[i];
patch[i * 4 + 1] = data[oplane + i];
patch[i * 4 + 2] = data[2 * oplane + i];
patch[i * 4 + 3] = 255;
}
try {
in0.dispose?.();
in1.dispose?.();
tensor.dispose?.();
for (const k in out) out[k]?.dispose?.();
} catch {}
patchCache = { key, patch: patch.slice() };
return patch;
} catch (err) {
// A model that cannot run is a soft repair, not a broken one — and this one
// line is what keeps the fallback diagnosable instead of invisible, because
// the picture that comes out of it looks like a workable repair.
console.warn('[inpaint] model unavailable, keeping the blurred fill', err);
return null;
}
})();
inFlightInpaint = runPromise;
try {
return await runPromise;
} finally {
if (inFlightInpaint === runPromise) {
inFlightInpaint = null;
}
}
}
@@ -273,7 +341,7 @@ export async function inpaintCrop(
// uses it to cut the crop it hands over: a 3000px box read back at 3000px and
// scaled here would be 27MB of pixels moved for a repair that ends up 512 wide.
export function inpaintCropSize(width: number, height: number): { w: number; h: number } {
const maxSide = isIOS() ? 384 : INPAINT_MAX_SIDE;
const maxSide = isMobile() ? 256 : INPAINT_MAX_SIDE;
const scale = Math.min(1, maxSide / Math.max(1, Math.max(width, height)));
return {
w: Math.max(8, Math.floor((Math.max(1, Math.round(width)) * scale) / 8) * 8),