3dtours/RecipesCam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(studio): stroke clustering for HEAL, mobile WebGL inpaint stability, and reorganize LIGHT panels with HSL

Browse Source
This commit is contained in:
3dtours
2026-10-14 12:57:31 +07:00
parent c1d098e4e2
commit 70ed61b8de
6 changed files with 333 additions and 222 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/frontend/src/engine/exportEngine.ts
+126 -82
View File
@@ -1300,6 +1300,54 @@ export async function renderPhoto(input: RenderInput): Promise<RenderResult | nu
});
}
function clusterHealSpots(
spots: { x: number; y: number; r: number }[],
width: number,
height: number
): { x: number; y: number; r: number }[][] {
if (spots.length <= 1) return spots.length ? [spots] : [];
const n = spots.length;
const adj: number[][] = Array.from({ length: n }, () => []);
for (let i = 0; i < n; i++) {
const si = spots[i];
const xi = si.x * width;
const yi = si.y * height;
const ri = si.r * width;
for (let j = i + 1; j < n; j++) {
const sj = spots[j];
const xj = sj.x * width;
const yj = sj.y * height;
const rj = sj.r * width;
const maxDist = (ri + rj) * 1.5;
const distSq = (xi - xj) ** 2 + (yi - yj) ** 2;
if (distSq <= maxDist * maxDist) {
adj[i].push(j);
adj[j].push(i);
}
}
}
const visited = new Uint8Array(n);
const clusters: { x: number; y: number; r: number }[][] = [];
for (let i = 0; i < n; i++) {
if (visited[i]) continue;
visited[i] = 1;
const cluster: { x: number; y: number; r: number }[] = [spots[i]];
const queue = [i];
while (queue.length > 0) {
const curr = queue.pop()!;
for (const neighbor of adj[curr]) {
if (!visited[neighbor]) {
visited[neighbor] = 1;
cluster.push(spots[neighbor]);
queue.push(neighbor);
}
}
}
clusters.push(cluster);
}
return clusters;
}
// 6e. HEAL — the FX tab's dust brush (see heal.ts). It is the last thing
// that touches the photo's own pixels, and deliberately so: the fill is made
// of the pixels the pipeline has already built, so a repair matches the
@@ -1424,6 +1472,7 @@ export async function renderPhoto(input: RenderInput): Promise<RenderResult | nu
// off on a smooth sky and the wrong colour outright on anything with shape.
const box = Skia.XYWHRect(boxLeft, boxTop, boxRight - boxLeft, boxBottom - boxTop);
const onto = Skia.XYWHRect(0, 0, boxRight - boxLeft, boxBottom - boxTop);
fillCanvas.clear(Skia.Color('transparent'));
if (cutImage) {
for (let k = levelCount; k >= 1; k--) {
const sigma = HEAL_FILL_BLUR * widest * 2 ** (k - 1);
@@ -1438,91 +1487,86 @@ export async function renderPhoto(input: RenderInput): Promise<RenderResult | nu
// The other half of the fill, and the better one where it can be had: the
// pixels under the brush invented by MI-GAN from the pixels around them
// (engine/inpaint.ts) rather than averaged from the rim. The blur above is
// still built first and still covers the whole box — it is drawn over by
// this where this reaches, and stands on its own wherever the model cannot
// be had (no runtime, no adapter, a load that fails): a soft repair beats
// no repair.
// (engine/inpaint.ts) rather than averaged from the rim.
//
// What the model is handed is the mask's own box with a margin of context
// around it, read back at the resolution the model will actually run at —
// a repair is the mask and a few brush widths of what borders it, and the
// pixels outside the mask are never read back out of the answer, so the
// crop is cut to the mask and not to the frame. It is drawn into the fill
// surface at its own place in the box, 1:1 whenever no downscale happened
// and scaled back up when one did.
const mx0 = Math.min(...xs) - widest;
const my0 = Math.min(...ys) - widest;
const mx1 = Math.max(...xs) + widest;
const my1 = Math.max(...ys) + widest;
const margin = Math.max(16, Math.min(64, Math.round(Math.max(widest * 1.5, 0.25 * Math.max(mx1 - mx0, my1 - my0)))));
const srcLeft = Math.max(boxLeft, Math.floor(mx0 - margin));
const srcTop = Math.max(boxTop, Math.floor(my0 - margin));
const srcRight = Math.min(boxRight, Math.ceil(mx1 + margin));
const srcBottom = Math.min(boxBottom, Math.ceil(my1 + margin));
const srcW = srcRight - srcLeft;
const srcH = srcBottom - srcTop;
if (srcW >= 8 && srcH >= 8) {
const { w: mw, h: mh } = inpaintCropSize(srcW, srcH);
const cropSurface = own(createSurface(mw, mh));
if (cropSurface) {
cropSurface.getCanvas().drawImageRect(
snap,
Skia.XYWHRect(srcLeft, srcTop, srcW, srcH),
Skia.XYWHRect(0, 0, mw, mh),
own(Skia.Paint())
);
flush(cropSurface);
const cropPixels = cropSurface.getCanvas().readPixels(0, 0, {
width: mw,
height: mh,
colorType: Skia.ColorType.RGBA_8888,
alphaType: Skia.AlphaType.Unpremul,
colorSpace: Skia.ColorSpace.SRGB,
});
if (cropPixels) {
// The brush's circles, at the crop's own resolution: the union of
// the spots, hard-edged. Built from the numbers and not read back
// out of `maskSurface`, because the two are not the same mask — the
// feathered one is the *composite's* weight, and a fade is not
// something the model has any use for. What the model wants is a
// hole, and it is told about it the way MI-GAN reads it: non-zero
// means keep (see inpaint.ts).
const kx = mw / srcW;
const ky = mh / srcH;
const hole = new Uint8Array(mw * mh);
for (const s of heal) {
const cx = (s.x * width - srcLeft) * kx;
const cy = (s.y * height - srcTop) * ky;
const rad = s.r * width * kx;
const x0 = Math.max(0, Math.floor(cx - rad));
const x1 = Math.min(mw - 1, Math.ceil(cx + rad));
const y0 = Math.max(0, Math.floor(cy - rad));
const y1 = Math.min(mh - 1, Math.ceil(cy + rad));
for (let y = y0; y <= y1; y++) {
const dy = y - cy;
const row = y * mw;
for (let x = x0; x <= x1; x++) {
const dx = x - cx;
if (dx * dx + dy * dy <= rad * rad) hole[row + x] = 1;
// Strokes are clustered so that separate brush repairs (e.g. Brush 1 and Brush 2)
// are solved independently at their own high resolution rather than lumped into
// one downscaled bounding box that would degrade previous repairs.
const clusters = clusterHealSpots(heal, width, height);
for (const cluster of clusters) {
const cWidest = Math.max(...cluster.map((s) => s.r * width));
const cXs = cluster.map((s) => s.x * width);
const cYs = cluster.map((s) => s.y * height);
const mx0 = Math.min(...cXs) - cWidest;
const my0 = Math.min(...cYs) - cWidest;
const mx1 = Math.max(...cXs) + cWidest;
const my1 = Math.max(...cYs) + cWidest;
const margin = Math.max(
16,
Math.min(64, Math.round(Math.max(cWidest * 1.5, 0.25 * Math.max(mx1 - mx0, my1 - my0))))
);
const srcLeft = Math.max(boxLeft, Math.floor(mx0 - margin));
const srcTop = Math.max(boxTop, Math.floor(my0 - margin));
const srcRight = Math.min(boxRight, Math.ceil(mx1 + margin));
const srcBottom = Math.min(boxBottom, Math.ceil(my1 + margin));
const srcW = srcRight - srcLeft;
const srcH = srcBottom - srcTop;
if (srcW >= 8 && srcH >= 8) {
const { w: mw, h: mh } = inpaintCropSize(srcW, srcH);
const cropSurface = own(createSurface(mw, mh));
if (cropSurface) {
cropSurface.getCanvas().drawImageRect(
snap,
Skia.XYWHRect(srcLeft, srcTop, srcW, srcH),
Skia.XYWHRect(0, 0, mw, mh),
own(Skia.Paint())
);
flush(cropSurface);
const cropPixels = cropSurface.getCanvas().readPixels(0, 0, {
width: mw,
height: mh,
colorType: Skia.ColorType.RGBA_8888,
alphaType: Skia.AlphaType.Unpremul,
colorSpace: Skia.ColorSpace.SRGB,
});
if (cropPixels) {
const kx = mw / srcW;
const ky = mh / srcH;
const hole = new Uint8Array(mw * mh);
for (const s of cluster) {
const cx = (s.x * width - srcLeft) * kx;
const cy = (s.y * height - srcTop) * ky;
const rad = s.r * width * kx;
const x0 = Math.max(0, Math.floor(cx - rad));
const x1 = Math.min(mw - 1, Math.ceil(cx + rad));
const y0 = Math.max(0, Math.floor(cy - rad));
const y1 = Math.min(mh - 1, Math.ceil(cy + rad));
for (let y = y0; y <= y1; y++) {
const dy = y - cy;
const row = y * mw;
for (let x = x0; x <= x1; x++) {
const dx = x - cx;
if (dx * dx + dy * dy <= rad * rad) hole[row + x] = 1;
}
}
}
}
const patch = await inpaintCrop(cropPixels, mw, mh, hole).catch(() => null);
if (patch) {
const patchSurface = own(createSurface(mw, mh));
if (patchSurface && patchSurface.getCanvas().writePixels(patch, mw, mh, 0, 0)) {
flush(patchSurface);
const patchImage = own(patchSurface.makeImageSnapshot());
if (patchImage) {
const patchPaint = own(Skia.Paint());
patchPaint.setAntiAlias(true);
fillCanvas.drawImageRect(
patchImage,
Skia.XYWHRect(0, 0, mw, mh),
Skia.XYWHRect(srcLeft - boxLeft, srcTop - boxTop, srcW, srcH),
patchPaint
);
const patch = await inpaintCrop(cropPixels, mw, mh, hole).catch(() => null);
if (patch) {
const patchSurface = own(createSurface(mw, mh));
if (patchSurface && patchSurface.getCanvas().writePixels(patch, mw, mh, 0, 0)) {
flush(patchSurface);
const patchImage = own(patchSurface.makeImageSnapshot());
if (patchImage) {
const patchPaint = own(Skia.Paint());
patchPaint.setAntiAlias(true);
fillCanvas.drawImageRect(
patchImage,
Skia.XYWHRect(0, 0, mw, mh),
Skia.XYWHRect(srcLeft - boxLeft, srcTop - boxTop, srcW, srcH),
patchPaint
);
}
}
}
}