3dtours/RecipesCam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(studio): stroke clustering for HEAL, mobile WebGL inpaint stability, and reorganize LIGHT panels with HSL

Browse Source
This commit is contained in:
3dtours
2026-10-14 12:57:31 +07:00
parent c1d098e4e2
commit 70ed61b8de
6 changed files with 333 additions and 222 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/frontend/src/engine/exportEngine.ts
+126 -82
View File
@@ -1300,6 +1300,54 @@ export async function renderPhoto(input: RenderInput): Promise<RenderResult | nu
});
}
function clusterHealSpots(
spots: { x: number; y: number; r: number }[],
width: number,
height: number
): { x: number; y: number; r: number }[][] {
if (spots.length <= 1) return spots.length ? [spots] : [];
const n = spots.length;
const adj: number[][] = Array.from({ length: n }, () => []);
for (let i = 0; i < n; i++) {
const si = spots[i];
const xi = si.x * width;
const yi = si.y * height;
const ri = si.r * width;
for (let j = i + 1; j < n; j++) {
const sj = spots[j];
const xj = sj.x * width;
const yj = sj.y * height;
const rj = sj.r * width;
const maxDist = (ri + rj) * 1.5;
const distSq = (xi - xj) ** 2 + (yi - yj) ** 2;
if (distSq <= maxDist * maxDist) {
adj[i].push(j);
adj[j].push(i);
}
}
}
const visited = new Uint8Array(n);
const clusters: { x: number; y: number; r: number }[][] = [];
for (let i = 0; i < n; i++) {
if (visited[i]) continue;
visited[i] = 1;
const cluster: { x: number; y: number; r: number }[] = [spots[i]];
const queue = [i];
while (queue.length > 0) {
const curr = queue.pop()!;
for (const neighbor of adj[curr]) {
if (!visited[neighbor]) {
visited[neighbor] = 1;
cluster.push(spots[neighbor]);
queue.push(neighbor);
}
}
}
clusters.push(cluster);
}
return clusters;
}
// 6e. HEAL — the FX tab's dust brush (see heal.ts). It is the last thing
// that touches the photo's own pixels, and deliberately so: the fill is made
// of the pixels the pipeline has already built, so a repair matches the
@@ -1424,6 +1472,7 @@ export async function renderPhoto(input: RenderInput): Promise<RenderResult | nu
// off on a smooth sky and the wrong colour outright on anything with shape.
const box = Skia.XYWHRect(boxLeft, boxTop, boxRight - boxLeft, boxBottom - boxTop);
const onto = Skia.XYWHRect(0, 0, boxRight - boxLeft, boxBottom - boxTop);
fillCanvas.clear(Skia.Color('transparent'));
if (cutImage) {
for (let k = levelCount; k >= 1; k--) {
const sigma = HEAL_FILL_BLUR * widest * 2 ** (k - 1);
@@ -1438,91 +1487,86 @@ export async function renderPhoto(input: RenderInput): Promise<RenderResult | nu
// The other half of the fill, and the better one where it can be had: the
// pixels under the brush invented by MI-GAN from the pixels around them
// (engine/inpaint.ts) rather than averaged from the rim. The blur above is
// still built first and still covers the whole box — it is drawn over by
// this where this reaches, and stands on its own wherever the model cannot
// be had (no runtime, no adapter, a load that fails): a soft repair beats
// no repair.
// (engine/inpaint.ts) rather than averaged from the rim.
//
// What the model is handed is the mask's own box with a margin of context
// around it, read back at the resolution the model will actually run at —
// a repair is the mask and a few brush widths of what borders it, and the
// pixels outside the mask are never read back out of the answer, so the
// crop is cut to the mask and not to the frame. It is drawn into the fill
// surface at its own place in the box, 1:1 whenever no downscale happened
// and scaled back up when one did.
const mx0 = Math.min(...xs) - widest;
const my0 = Math.min(...ys) - widest;
const mx1 = Math.max(...xs) + widest;
const my1 = Math.max(...ys) + widest;
const margin = Math.max(16, Math.min(64, Math.round(Math.max(widest * 1.5, 0.25 * Math.max(mx1 - mx0, my1 - my0)))));
const srcLeft = Math.max(boxLeft, Math.floor(mx0 - margin));
const srcTop = Math.max(boxTop, Math.floor(my0 - margin));
const srcRight = Math.min(boxRight, Math.ceil(mx1 + margin));
const srcBottom = Math.min(boxBottom, Math.ceil(my1 + margin));
const srcW = srcRight - srcLeft;
const srcH = srcBottom - srcTop;
if (srcW >= 8 && srcH >= 8) {
const { w: mw, h: mh } = inpaintCropSize(srcW, srcH);
const cropSurface = own(createSurface(mw, mh));
if (cropSurface) {
cropSurface.getCanvas().drawImageRect(
snap,
Skia.XYWHRect(srcLeft, srcTop, srcW, srcH),
Skia.XYWHRect(0, 0, mw, mh),
own(Skia.Paint())
);
flush(cropSurface);
const cropPixels = cropSurface.getCanvas().readPixels(0, 0, {
width: mw,
height: mh,
colorType: Skia.ColorType.RGBA_8888,
alphaType: Skia.AlphaType.Unpremul,
colorSpace: Skia.ColorSpace.SRGB,
});
if (cropPixels) {
// The brush's circles, at the crop's own resolution: the union of
// the spots, hard-edged. Built from the numbers and not read back
// out of `maskSurface`, because the two are not the same mask — the
// feathered one is the *composite's* weight, and a fade is not
// something the model has any use for. What the model wants is a
// hole, and it is told about it the way MI-GAN reads it: non-zero
// means keep (see inpaint.ts).
const kx = mw / srcW;
const ky = mh / srcH;
const hole = new Uint8Array(mw * mh);
for (const s of heal) {
const cx = (s.x * width - srcLeft) * kx;
const cy = (s.y * height - srcTop) * ky;
const rad = s.r * width * kx;
const x0 = Math.max(0, Math.floor(cx - rad));
const x1 = Math.min(mw - 1, Math.ceil(cx + rad));
const y0 = Math.max(0, Math.floor(cy - rad));
const y1 = Math.min(mh - 1, Math.ceil(cy + rad));
for (let y = y0; y <= y1; y++) {
const dy = y - cy;
const row = y * mw;
for (let x = x0; x <= x1; x++) {
const dx = x - cx;
if (dx * dx + dy * dy <= rad * rad) hole[row + x] = 1;
// Strokes are clustered so that separate brush repairs (e.g. Brush 1 and Brush 2)
// are solved independently at their own high resolution rather than lumped into
// one downscaled bounding box that would degrade previous repairs.
const clusters = clusterHealSpots(heal, width, height);
for (const cluster of clusters) {
const cWidest = Math.max(...cluster.map((s) => s.r * width));
const cXs = cluster.map((s) => s.x * width);
const cYs = cluster.map((s) => s.y * height);
const mx0 = Math.min(...cXs) - cWidest;
const my0 = Math.min(...cYs) - cWidest;
const mx1 = Math.max(...cXs) + cWidest;
const my1 = Math.max(...cYs) + cWidest;
const margin = Math.max(
16,
Math.min(64, Math.round(Math.max(cWidest * 1.5, 0.25 * Math.max(mx1 - mx0, my1 - my0))))
);
const srcLeft = Math.max(boxLeft, Math.floor(mx0 - margin));
const srcTop = Math.max(boxTop, Math.floor(my0 - margin));
const srcRight = Math.min(boxRight, Math.ceil(mx1 + margin));
const srcBottom = Math.min(boxBottom, Math.ceil(my1 + margin));
const srcW = srcRight - srcLeft;
const srcH = srcBottom - srcTop;
if (srcW >= 8 && srcH >= 8) {
const { w: mw, h: mh } = inpaintCropSize(srcW, srcH);
const cropSurface = own(createSurface(mw, mh));
if (cropSurface) {
cropSurface.getCanvas().drawImageRect(
snap,
Skia.XYWHRect(srcLeft, srcTop, srcW, srcH),
Skia.XYWHRect(0, 0, mw, mh),
own(Skia.Paint())
);
flush(cropSurface);
const cropPixels = cropSurface.getCanvas().readPixels(0, 0, {
width: mw,
height: mh,
colorType: Skia.ColorType.RGBA_8888,
alphaType: Skia.AlphaType.Unpremul,
colorSpace: Skia.ColorSpace.SRGB,
});
if (cropPixels) {
const kx = mw / srcW;
const ky = mh / srcH;
const hole = new Uint8Array(mw * mh);
for (const s of cluster) {
const cx = (s.x * width - srcLeft) * kx;
const cy = (s.y * height - srcTop) * ky;
const rad = s.r * width * kx;
const x0 = Math.max(0, Math.floor(cx - rad));
const x1 = Math.min(mw - 1, Math.ceil(cx + rad));
const y0 = Math.max(0, Math.floor(cy - rad));
const y1 = Math.min(mh - 1, Math.ceil(cy + rad));
for (let y = y0; y <= y1; y++) {
const dy = y - cy;
const row = y * mw;
for (let x = x0; x <= x1; x++) {
const dx = x - cx;
if (dx * dx + dy * dy <= rad * rad) hole[row + x] = 1;
}
}
}
}
const patch = await inpaintCrop(cropPixels, mw, mh, hole).catch(() => null);
if (patch) {
const patchSurface = own(createSurface(mw, mh));
if (patchSurface && patchSurface.getCanvas().writePixels(patch, mw, mh, 0, 0)) {
flush(patchSurface);
const patchImage = own(patchSurface.makeImageSnapshot());
if (patchImage) {
const patchPaint = own(Skia.Paint());
patchPaint.setAntiAlias(true);
fillCanvas.drawImageRect(
patchImage,
Skia.XYWHRect(0, 0, mw, mh),
Skia.XYWHRect(srcLeft - boxLeft, srcTop - boxTop, srcW, srcH),
patchPaint
);
const patch = await inpaintCrop(cropPixels, mw, mh, hole).catch(() => null);
if (patch) {
const patchSurface = own(createSurface(mw, mh));
if (patchSurface && patchSurface.getCanvas().writePixels(patch, mw, mh, 0, 0)) {
flush(patchSurface);
const patchImage = own(patchSurface.makeImageSnapshot());
if (patchImage) {
const patchPaint = own(Skia.Paint());
patchPaint.setAntiAlias(true);
fillCanvas.drawImageRect(
patchImage,
Skia.XYWHRect(0, 0, mw, mh),
Skia.XYWHRect(srcLeft - boxLeft, srcTop - boxTop, srcW, srcH),
patchPaint
);
}
}
}
}
docker/frontend/src/engine/inpaint.ts
+19 -45
View File
@@ -97,11 +97,12 @@ async function warmsUp(ort: typeof Ort, session: Ort.InferenceSession): Promise<
}
// Whether the browser has a device at all.
// WebGPU on iOS (iPhone / iPad / WKWebView) is experimental and unstable;
// compiling WGSL or dispatching ONNX pipelines crashes WebKit's GPUProcess,
// causing iOS to silently reload the page. iOS is therefore excluded from WebGPU.
// Whether the browser has a device at all.
// Mobile devices (Android Chrome and iOS WebKit) have unified memory architectures where
// WebGPU buffer allocation destroys WebGL contexts (CanvasKit GrDirectContext loss -> black screen).
// Therefore WebGPU is strictly restricted to desktop.
async function hasDevice(): Promise<boolean> {
if (isIOS()) return false;
if (isMobile()) return false;
try {
return Boolean(await navigator.gpu?.requestAdapter());
} catch {
@@ -140,10 +141,8 @@ function load(): Promise<Loaded> {
: self.crossOriginIsolated
? Math.min(4, navigator.hardwareConcurrency || 1)
: 1;
// ALWAYS proxy ONNX WASM execution into a Web Worker.
// Running heavy WASM inferences on the main thread locks the UI loop ("not responding")
// and triggers browser watchdog termination (crash / full reload).
ort.env.wasm.proxy = true;
// On mobile, proxy=false avoids launching multiple WASM worker heaps and iOS Jetsam kills.
ort.env.wasm.proxy = !mobile;
ort.env.webgpu.powerPreference = 'high-performance';
const session = await openSession(ort);
return { ort, session };
@@ -158,28 +157,17 @@ function load(): Promise<Loaded> {
// Start fetching the model before the brush that needs it, so the visitor waits
// for the repair and not for the download. Called when HEAL is switched on.
//
// The bytes, and only the bytes: importing the runtime starts its thread pool —
// a Web Worker each, a copy of the 27MB binary each — and a pool that then
// belongs to the page until it dies, whether or not a stroke is ever painted.
// Switching a tool on is not the work; the download is, and the browser's cache
// is what keeps it.
export function preloadInpaint(): void {
if (isIOS()) return;
for (const file of [...RUNTIME_FILES.map((f) => WASM_DIR + f), MODEL_URL]) {
void fetch(file).catch(() => {});
}
}
interface CachedPatch {
key: string;
patch: Uint8Array;
}
let patchCache: CachedPatch | null = null;
const patchCache = new Map<string, Uint8Array>();
let inFlightInpaint: Promise<Uint8Array | null> | null = null;
export function clearInpaintCache(): void {
patchCache = null;
patchCache.clear();
}
// The page is going away, or coming back from the back/forward cache: hand the
@@ -188,7 +176,7 @@ export function clearInpaintCache(): void {
// back loads the runtime again instead of calling into a released session — the
// files are cached, so that is a second.
export async function releaseInpaint(): Promise<void> {
patchCache = null;
patchCache.clear();
const pending = loaded;
loaded = null;
if (!pending) return;
@@ -206,30 +194,12 @@ if (typeof addEventListener === 'function') {
}
// The pixels under the brush, invented from the pixels around them.
//
// `rgba` is the crop the renderer cut out of the frame, at its own resolution;
// `hole` is one byte per pixel, non-zero for the pixels the brush painted. The
// answer is the same crop in RGBA at the same size, or null when the model could
// not be had — the caller keeps its blur fill in that case, and a repair that is
// merely soft beats no repair at all.
//
// A crop larger than INPAINT_MAX_SIDE on its longest side is scaled down to fit
// and the answer is scaled back up by the caller. A brush stroke is a soft shape
// made of soft shapes; what a factor of two of resample costs it is nothing
// beside the seconds it saves.
export async function inpaintCrop(
rgba: Uint8Array | Uint8ClampedArray,
width: number,
height: number,
hole: Uint8Array
): Promise<Uint8Array | null> {
// On iOS (iPhone/iPad WebKit), executing a 27MB deep learning model in WebAssembly
// alongside CanvasKit exceeds the strict iOS per-tab Jetsam memory limit (~300MB),
// causing iOS kernel to SIGKILL the WebProcess (tab crash / reload / "không thể mở trang này").
// Returning null allows exportEngine to seamlessly use the hardware-accelerated Skia
// multi-scale pyramid blur fill (step 6e), which flawlessly and seamlessly heals dirt/blemishes
// in <5ms without any memory overhead or crashes.
if (isIOS()) return null;
const maxSide = isMobile() ? 256 : INPAINT_MAX_SIDE;
let w = Math.max(8, Math.round(width));
@@ -259,8 +229,8 @@ export async function inpaintCrop(
}
const key = `${w}x${h}_${holeCount}_${holeHash}_${pixHash}`;
if (patchCache && patchCache.key === key) {
return patchCache.patch.slice();
if (patchCache.has(key)) {
return patchCache.get(key)!.slice();
}
// Single-flight lock: serialize inference executions to prevent WebAssembly memory spikes on mobile
@@ -268,8 +238,8 @@ export async function inpaintCrop(
try {
await inFlightInpaint;
} catch {}
if (patchCache && patchCache.key === key) {
return patchCache.patch.slice();
if (patchCache.has(key)) {
return patchCache.get(key)!.slice();
}
}
@@ -324,7 +294,11 @@ export async function inpaintCrop(
for (const k in out) out[k]?.dispose?.();
} catch {}
patchCache = { key, patch: patch.slice() };
if (patchCache.size > 50) {
const first = patchCache.keys().next().value;
if (first) patchCache.delete(first);
}
patchCache.set(key, patch.slice());
return patch;
} catch (err) {
// A model that cannot run is a soft repair, not a broken one — and this one