3dtours/RecipesCam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(heal): apply HEAL directly to base image before exposure grading and use fast texture synthesis on mobile to prevent iOS reload

Browse Source
This commit is contained in:
3dtours
2026-10-16 12:04:18 +07:00
parent d579a67ed9
commit f96a60f416
3 changed files with 514 additions and 447 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/frontend/src/engine/exportEngine.ts
+380 -418
View File
@@ -56,7 +56,7 @@ import {
HEAL_PATCH_STOPS,
readHeal,
} from '../../shared/utils/heal';
import { inpaintCrop, inpaintCropSize, INPAINT_REF_LONG } from './inpaint';
import { inpaintCrop, inpaintCropSize, INPAINT_REF_LONG, onClearInpaint } from './inpaint';
import { mosaicSkSL, mosaicUniforms, readMosaic } from '../../shared/utils/mosaic';
import { gradientMaskSkSL, maskUniforms, readMasks, masksHaveSpatial } from '../../shared/utils/gradientMask';
import {
@@ -873,6 +873,370 @@ function loadOldFilm(assets: RenderAssets) {
return oldFilmImage;
}
function clusterHealSpots(
spots: { x: number; y: number; r: number; stroke?: number }[],
width: number,
height: number
): { x: number; y: number; r: number; stroke?: number }[][] {
if (spots.length <= 1) return spots.length ? [spots] : [];
const n = spots.length;
const adj: number[][] = Array.from({ length: n }, () => []);
for (let i = 0; i < n; i++) {
const si = spots[i];
const xi = si.x * width;
const yi = si.y * height;
const ri = si.r * width;
for (let j = i + 1; j < n; j++) {
const sj = spots[j];
if (si.stroke !== sj.stroke) continue;
const xj = sj.x * width;
const yj = sj.y * height;
const rj = sj.r * width;
const maxDist = (ri + rj) * 1.5;
const distSq = (xi - xj) ** 2 + (yi - yj) ** 2;
if (distSq <= maxDist * maxDist) {
adj[i].push(j);
adj[j].push(i);
}
}
}
const isMob = typeof navigator !== 'undefined' && /iPad|iPhone|iPod|Android|Mobile/i.test(navigator.userAgent || '');
const maxSpan = isMob ? 600 : 1024;
const visited = new Uint8Array(n);
const clusters: { x: number; y: number; r: number; stroke?: number }[][] = [];
for (let i = 0; i < n; i++) {
if (visited[i]) continue;
visited[i] = 1;
const cluster: { x: number; y: number; r: number; stroke?: number }[] = [spots[i]];
let minX = spots[i].x * width;
let maxX = minX;
let minY = spots[i].y * height;
let maxY = minY;
const queue = [i];
while (queue.length > 0) {
const curr = queue.shift()!;
for (const neighbor of adj[curr]) {
if (!visited[neighbor]) {
const nx = spots[neighbor].x * width;
const ny = spots[neighbor].y * height;
const newMinX = Math.min(minX, nx);
const newMaxX = Math.max(maxX, nx);
const newMinY = Math.min(minY, ny);
const newMaxY = Math.max(maxY, ny);
if (newMaxX - newMinX <= maxSpan && newMaxY - newMinY <= maxSpan) {
visited[neighbor] = 1;
cluster.push(spots[neighbor]);
minX = newMinX;
maxX = newMaxX;
minY = newMinY;
maxY = newMaxY;
queue.push(neighbor);
}
}
}
}
clusters.push(cluster);
}
return clusters;
}
interface HealedBaseCache {
baseImage: any;
healKey: string;
width: number;
height: number;
healedSnapshot: any;
}
let cachedHealedBase: HealedBaseCache | null = null;
export function clearHealBaseCache(): void {
if (cachedHealedBase?.healedSnapshot) {
try {
cachedHealedBase.healedSnapshot.dispose?.();
} catch {}
}
cachedHealedBase = null;
}
onClearInpaint(clearHealBaseCache);
async function applyHealToBase(
baseImage: any,
healSpots: { x: number; y: number; r: number; stroke?: number }[],
width: number,
height: number
): Promise<any> {
const healKey = healSpots
.map((s) => `${s.x.toFixed(4)},${s.y.toFixed(4)},${s.r.toFixed(4)},${s.stroke ?? 0}`)
.join(';');
if (
cachedHealedBase &&
cachedHealedBase.baseImage === baseImage &&
cachedHealedBase.healKey === healKey &&
cachedHealedBase.width === width &&
cachedHealedBase.height === height
) {
return cachedHealedBase.healedSnapshot;
}
let curImage = baseImage;
const tempOwned: any[] = [];
const tempOwn = <T,>(obj: T | null | undefined): T => {
if (obj) tempOwned.push(obj);
return obj as T;
};
try {
const { healCutEffect: cutEffect, healFillEffect: fillEffect } = effects();
const clusters = clusterHealSpots(healSpots, width, height);
for (const heal of clusters) {
const snap = curImage;
const maskSurface = tempOwn(createSurface(width, height));
if (!maskSurface) continue;
const maskCanvas = maskSurface.getCanvas();
maskCanvas.clear(Skia.Color('transparent'));
for (const s of heal) {
const cx = s.x * width;
const cy = s.y * height;
const rad = s.r * width;
const shader = tempOwn(
Skia.Shader.MakeRadialGradient(
[cx, cy],
rad,
new Float32Array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0]),
HEAL_MASK_STOPS,
Skia.TileMode.Clamp
)
);
const paint = tempOwn(Skia.Paint());
paint.setAntiAlias(true);
paint.setShader(shader);
maskCanvas.drawCircle(cx, cy, rad, paint);
}
flush(maskSurface);
const maskImage = tempOwn(maskSurface.makeImageSnapshot());
const cutSurface = tempOwn(createSurface(width, height));
if (snap && maskImage && cutEffect && cutSurface) {
const cutShader = tempOwn(
cutEffect.makeShaderWithChildren([], [tempOwn(imageShaderChild(snap)), tempOwn(imageShaderChild(maskImage))])
);
if (cutShader) {
const cutCanvas = cutSurface.getCanvas();
cutCanvas.clear(Skia.Color('transparent'));
const cutPaint = tempOwn(Skia.Paint());
cutPaint.setShader(cutShader);
cutCanvas.drawRect(Skia.XYWHRect(0, 0, width, height), cutPaint);
}
}
flush(cutSurface);
const cutImage = tempOwn(cutSurface.makeImageSnapshot());
const widest = Math.max(...heal.map((s) => s.r * width));
const xs = heal.map((s) => s.x * width);
const ys = heal.map((s) => s.y * height);
const span = Math.hypot(Math.max(...xs) - Math.min(...xs), Math.max(...ys) - Math.min(...ys));
const levelCount = Math.min(
HEAL_FILL_MAX_LEVELS,
Math.max(2, 1 + Math.ceil(Math.log2(2 + span / 2 / widest)))
);
const pad = Math.max(64, Math.min(512, Math.round(widest * 3 + span * 0.6)));
const boxLeft = Math.max(0, Math.floor(Math.min(...xs) - widest - pad));
const boxTop = Math.max(0, Math.floor(Math.min(...ys) - widest - pad));
const boxRight = Math.min(width, Math.ceil(Math.max(...xs) + widest + pad));
const boxBottom = Math.min(height, Math.ceil(Math.max(...ys) + widest + pad));
const fillSurface = tempOwn(createSurface(boxRight - boxLeft, boxBottom - boxTop));
if (!fillSurface) continue;
const fillCanvas = fillSurface.getCanvas();
const box = Skia.XYWHRect(boxLeft, boxTop, boxRight - boxLeft, boxBottom - boxTop);
const onto = Skia.XYWHRect(0, 0, boxRight - boxLeft, boxBottom - boxTop);
fillCanvas.clear(Skia.Color('transparent'));
if (cutImage) {
for (let k = levelCount; k >= 1; k--) {
const sigma = HEAL_FILL_BLUR * widest * 2 ** (k - 1);
const blurPaint = tempOwn(Skia.Paint());
blurPaint.setImageFilter(tempOwn(Skia.ImageFilter.MakeBlur(sigma, sigma, Skia.TileMode.Clamp, null)));
fillCanvas.drawImageRect(cutImage, box, onto, blurPaint);
}
fillCanvas.drawImageRect(cutImage, box, onto, tempOwn(Skia.Paint()));
}
const subClusters = clusterHealSpots(heal, width, height);
for (const cluster of subClusters) {
const cWidest = Math.max(...cluster.map((s) => s.r * width));
const cXs = cluster.map((s) => s.x * width);
const cYs = cluster.map((s) => s.y * height);
const mx0 = Math.min(...cXs) - cWidest;
const my0 = Math.min(...cYs) - cWidest;
const mx1 = Math.max(...cXs) + cWidest;
const my1 = Math.max(...cYs) + cWidest;
const cSpanW = mx1 - mx0;
const cSpanH = my1 - my0;
const cSpan = Math.max(cSpanW, cSpanH);
const margin = Math.max(96, Math.min(512, Math.round(cWidest * 3 + cSpan * 0.8)));
const srcLeft = Math.max(boxLeft, Math.floor(mx0 - margin));
const srcTop = Math.max(boxTop, Math.floor(my0 - margin));
const srcRight = Math.min(boxRight, Math.ceil(mx1 + margin));
const srcBottom = Math.min(boxBottom, Math.ceil(my1 + margin));
const srcW = srcRight - srcLeft;
const srcH = srcBottom - srcTop;
if (srcW >= 8 && srcH >= 8) {
const { w: mw, h: mh } = inpaintCropSize(
(srcW / width) * INPAINT_REF_LONG,
(srcH / height) * INPAINT_REF_LONG
);
const cropSurface = tempOwn(createSurface(mw, mh));
if (cropSurface) {
cropSurface.getCanvas().drawImageRect(
snap,
Skia.XYWHRect(srcLeft, srcTop, srcW, srcH),
Skia.XYWHRect(0, 0, mw, mh),
tempOwn(Skia.Paint())
);
flush(cropSurface);
const cropPixels = cropSurface.getCanvas().readPixels(0, 0, {
width: mw,
height: mh,
colorType: Skia.ColorType.RGBA_8888,
alphaType: Skia.AlphaType.Unpremul,
colorSpace: Skia.ColorSpace.SRGB,
});
if (cropPixels) {
const kx = mw / srcW;
const ky = mh / srcH;
const hole = new Uint8Array(mw * mh);
for (const s of cluster) {
const cx = (s.x * width - srcLeft) * kx;
const cy = (s.y * height - srcTop) * ky;
const rad = s.r * width * kx;
const x0 = Math.max(0, Math.floor(cx - rad));
const x1 = Math.min(mw - 1, Math.ceil(cx + rad));
const y0 = Math.max(0, Math.floor(cy - rad));
const y1 = Math.min(mh - 1, Math.ceil(cy + rad));
for (let y = y0; y <= y1; y++) {
const dy = y - cy;
const row = y * mw;
for (let x = x0; x <= x1; x++) {
const dx = x - cx;
if (dx * dx + dy * dy <= rad * rad) hole[row + x] = 1;
}
}
}
const patchKey = `${mw}x${mh}_` + cluster.map((s) => `${s.x.toFixed(4)},${s.y.toFixed(4)},${s.r.toFixed(4)}`).join(';');
(hole as { patchKey?: string }).patchKey = patchKey;
const patch = await inpaintCrop(cropPixels, mw, mh, hole).catch(() => null);
if (patch) {
const patchSurface = tempOwn(createSurface(mw, mh));
if (patchSurface && patchSurface.getCanvas().writePixels(patch, mw, mh, 0, 0)) {
flush(patchSurface);
const patchImage = tempOwn(patchSurface.makeImageSnapshot());
if (patchImage) {
const patchMaskSurface = tempOwn(createSurface(srcW, srcH));
if (patchMaskSurface) {
const patchCanvas = patchMaskSurface.getCanvas();
patchCanvas.clear(Skia.Color('transparent'));
for (const s of cluster) {
const cx = s.x * width - srcLeft;
const cy = s.y * height - srcTop;
const rad = s.r * width;
const shader = tempOwn(
Skia.Shader.MakeRadialGradient(
[cx, cy],
rad,
new Float32Array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0]),
HEAL_PATCH_STOPS,
Skia.TileMode.Clamp
)
);
const spotPaint = tempOwn(Skia.Paint());
spotPaint.setAntiAlias(true);
spotPaint.setShader(shader);
patchCanvas.drawCircle(cx, cy, rad, spotPaint);
}
const patchPaint = tempOwn(Skia.Paint());
patchPaint.setAntiAlias(true);
patchPaint.setBlendMode(Skia.BlendMode.SrcIn);
patchCanvas.drawImageRect(
patchImage,
Skia.XYWHRect(0, 0, mw, mh),
Skia.XYWHRect(0, 0, srcW, srcH),
patchPaint
);
flush(patchMaskSurface);
const maskedPatch = tempOwn(patchMaskSurface.makeImageSnapshot());
if (maskedPatch) {
fillCanvas.drawImageRect(
maskedPatch,
Skia.XYWHRect(0, 0, srcW, srcH),
Skia.XYWHRect(srcLeft - boxLeft, srcTop - boxTop, srcW, srcH),
tempOwn(Skia.Paint())
);
}
}
}
}
}
}
}
}
}
flush(fillSurface);
const fillImage = tempOwn(fillSurface.makeImageSnapshot());
if (fillImage && snap && maskImage && fillEffect) {
const shader = tempOwn(
fillEffect.makeShaderWithChildren([], [
tempOwn(imageShaderChild(snap)),
tempOwn(imageShaderChildAt(fillImage, boxLeft, boxTop)),
tempOwn(imageShaderChild(maskImage)),
])
);
if (shader) {
const clusterSurface = tempOwn(createSurface(width, height));
if (clusterSurface) {
const clusterCanvas = clusterSurface.getCanvas();
clusterCanvas.clear(Skia.Color('transparent'));
const healPaint = tempOwn(Skia.Paint());
healPaint.setShader(shader);
clusterCanvas.drawRect(Skia.XYWHRect(0, 0, width, height), healPaint);
flush(clusterSurface);
const clusterSnap = clusterSurface.makeImageSnapshot();
if (clusterSnap) {
curImage = clusterSnap;
tempOwn(clusterSnap);
}
}
}
}
}
if (cachedHealedBase?.healedSnapshot && cachedHealedBase.healedSnapshot !== curImage) {
try { cachedHealedBase.healedSnapshot.dispose?.(); } catch {}
}
cachedHealedBase = {
baseImage,
healKey,
width,
height,
healedSnapshot: curImage,
};
return curImage;
} finally {
for (const obj of tempOwned) {
if (obj !== curImage) {
try { obj.dispose?.(); } catch {}
}
}
}
}
export async function renderPhoto(input: RenderInput): Promise<RenderResult | null> {
const { recipe, frameId, options } = input;
const owned: SkDisposable[] = [];
@@ -957,6 +1321,21 @@ export async function renderPhoto(input: RenderInput): Promise<RenderResult | nu
const adjustments = recipe.adjustments;
// 2b. Base photo HEAL: applied directly to skImage before color grading and exposure.
// In professional RAW engines (Lightroom, Camera Raw), spot healing operates on
// base pixels so that exposure and tone adjustments grade the healed area and
// surrounding pixels uniformly in exact unison, preventing patches from standing
// out when brightness is adjusted.
const healSpots = readHeal(adjustments.heal);
if (healSpots.length > 0) {
const healed = await applyHealToBase(skImage, healSpots, width, height);
if (healed && healed !== skImage) {
own(healed);
release(owned, skImage);
skImage = healed;
}
}
// 3. Colour matrix, then the exposure pass (+EV unless the camera already
// biased the shot). The matrix grades into its OWN image first, because a
// SkPaint runs its shader before its colourFilter: with both on one paint the
@@ -1383,423 +1762,6 @@ export async function renderPhoto(input: RenderInput): Promise<RenderResult | nu
});
}
function clusterHealSpots(
spots: { x: number; y: number; r: number; stroke?: number }[],
width: number,
height: number
): { x: number; y: number; r: number }[][] {
if (spots.length <= 1) return spots.length ? [spots] : [];
const n = spots.length;
const adj: number[][] = Array.from({ length: n }, () => []);
for (let i = 0; i < n; i++) {
const si = spots[i];
const xi = si.x * width;
const yi = si.y * height;
const ri = si.r * width;
for (let j = i + 1; j < n; j++) {
const sj = spots[j];
// Two drags are two repairs even where their circles touch. A stroke is
// HEALed when the user lets go of it, and what the solve below reads is
// the frame the earlier ones have already left — so a stroke next to a
// healed one is drawn from a photo that no longer holds the first, and
// never carries its detail into the second. One cluster spanning both
// would be a single crop of the pair, solved from the ground around the
// two: the detail of one repair coming back inside the other. Spots
// written before the brush numbered its drags have none, and stay
// grouped by distance, the reading they always got.
if (si.stroke !== sj.stroke) continue;
const xj = sj.x * width;
const yj = sj.y * height;
const rj = sj.r * width;
const maxDist = (ri + rj) * 1.5;
const distSq = (xi - xj) ** 2 + (yi - yj) ** 2;
if (distSq <= maxDist * maxDist) {
adj[i].push(j);
adj[j].push(i);
}
}
}
// Dimension of a cluster's bounding box. Allows unified processing of connected
// brush strokes in a single fast pass instead of slow fragmented passes.
const isMob = typeof navigator !== 'undefined' && /iPad|iPhone|iPod|Android|Mobile/i.test(navigator.userAgent || '');
const maxSpan = isMob ? 600 : 1024;
const visited = new Uint8Array(n);
const clusters: { x: number; y: number; r: number }[][] = [];
for (let i = 0; i < n; i++) {
if (visited[i]) continue;
visited[i] = 1;
const cluster: { x: number; y: number; r: number }[] = [spots[i]];
let minX = spots[i].x * width;
let maxX = minX;
let minY = spots[i].y * height;
let maxY = minY;
const queue = [i];
while (queue.length > 0) {
const curr = queue.shift()!;
for (const neighbor of adj[curr]) {
if (!visited[neighbor]) {
const nx = spots[neighbor].x * width;
const ny = spots[neighbor].y * height;
const newMinX = Math.min(minX, nx);
const newMaxX = Math.max(maxX, nx);
const newMinY = Math.min(minY, ny);
const newMaxY = Math.max(maxY, ny);
if (newMaxX - newMinX <= maxSpan && newMaxY - newMinY <= maxSpan) {
visited[neighbor] = 1;
cluster.push(spots[neighbor]);
minX = newMinX;
maxX = newMaxX;
minY = newMinY;
maxY = newMaxY;
queue.push(neighbor);
}
}
}
}
clusters.push(cluster);
}
return clusters;
}
// 6e. HEAL — the FX tab's dust brush (see heal.ts). It is the last thing
// that touches the photo's own pixels, and deliberately so: the fill is made
// of the pixels the pipeline has already built, so a repair matches the
// grade, the blur and the grain around it by construction instead of being a
// second render of the same look that has to agree. The card, the wall and
// the watermarks are drawn after it, so a healed spot still carries the
// frame's own edge.
//
// Three moves, in this order. The brush's circles become ONE mask — a union,
// which is what makes a stroke a single shape instead of a row of circles.
// The mask is cut out of the frame, leaving a hole with the photo's own
// pixels all around its rim. And the hole is filled from that rim, by a
// blur of the frame, so nothing is copied from anywhere: what was under the
// brush is the light of what borders it.
// ONE DRAG AT A TIME, in the order the user made them, each solved against
// the frame the ones before it have already left. This is what "HEALed"
// means: the repair a stroke left behind is part of the photo from then on,
// and the drag laid next to it is invented from a frame that no longer holds
// what the first drag took away. Solved as one list instead — which is what
// this did, and what the whole of it below still does, once per drag — the
// second brush reaches back into the first: the crop around the new stroke
// covers the old ones, so they are re-derived from a frame that still had
// their dust in it, and a repair the user has already looked at and accepted
// moves under them the moment they brush beside it.
//
// The cost is one pass of the block below per drag rather than one for the
// photo: on a photo with a handful of repairs that is a handful of masks and
// composites, and the box each fill is built on is still the stroke's own,
// so nothing here is proportional to the frame but the mask and the one draw
// that puts the repair back. ponytail: a photo with dozens of repairs pays
// for all of them on every preview; the ceiling is a cache of the frame as
// it stands after each drag, keyed by the list up to that drag, and it is
// worth its bookkeeping only if the preview is ever felt to lag.
const healSpots = readHeal(adjustments.heal);
for (const heal of healSpots.length ? clusterHealSpots(healSpots, width, height) : []) {
const { healCutEffect: cutEffect, healFillEffect: fillEffect } = effects();
flush(surface);
const snap = own(surface.makeImageSnapshot());
// The mask, at the frame's own resolution: one radial gradient per spot,
// solid out to HEAL_FEATHER of its radius and clear at its edge, each laid
// over the last — so two spots of one stroke are one shape with no seam
// between them, and the mask's own border is a fade rather than an outline
// the fill would have to match exactly.
const maskSurface = own(createSurface(width, height));
const maskCanvas = maskSurface.getCanvas();
maskCanvas.clear(Skia.Color('transparent'));
for (const s of heal) {
const cx = s.x * width;
const cy = s.y * height;
const rad = s.r * width;
const shader = own(
Skia.Shader.MakeRadialGradient(
[cx, cy],
rad,
new Float32Array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0]),
HEAL_MASK_STOPS,
Skia.TileMode.Clamp
)
);
const paint = own(Skia.Paint());
paint.setAntiAlias(true);
paint.setShader(shader);
maskCanvas.drawCircle(cx, cy, rad, paint);
}
flush(maskSurface);
const maskImage = own(maskSurface.makeImageSnapshot());
// The frame with the mask taken out of it: the hole the fill works on.
const cutSurface = own(createSurface(width, height));
if (snap && maskImage && cutEffect) {
const cutShader = own(
cutEffect.makeShaderWithChildren([], [own(imageShaderChild(snap)), own(imageShaderChild(maskImage))])
);
if (cutShader) {
const cutCanvas = cutSurface.getCanvas();
cutCanvas.clear(Skia.Color('transparent'));
const cutPaint = own(Skia.Paint());
cutPaint.setShader(cutShader);
cutCanvas.drawRect(Skia.XYWHRect(0, 0, width, height), cutPaint);
}
}
flush(cutSurface);
const cutImage = own(cutSurface.makeImageSnapshot());
// The fill: the cut frame blurred at a few widths, widest first, each
// finer one drawn over it, the untouched cut on top. A blur weighs every
// pixel by how much of the frame's light reaches it, and the hole has none
// of its own left to weigh — so the hole comes back as the light of its
// rim, carried in with no seam and nothing copied from anywhere. Widest
// first means the widest blur only has a say where every finer one has
// nothing (the middle of a hole deeper than a brush), and the untouched
// cut wins wherever the photo is still there, so the rim is the rim.
//
// Blurs and not a pyramid of halvings, on purpose. This Skia gets the
// premultiply wrong on a snapshot used as a SCALED draw source — its rgb
// comes back with the alpha multiplied in a second time — and a pyramid is
// built of exactly those halvings; a partly transparent rim then came out
// a wash of grey, which is what the heal was painting over the photo.
// Every draw here is 1:1, which filters correctly (measured, see
// heal-check.mjs).
//
// How many levels: the finest is HEAL_FILL_BLUR brushes wide and each one
// is twice the last, so n of them reach HEAL_FILL_BLUR * 2^(n-1) radii —
// enough to span half the stroke's own diagonal from either end of it.
// Fewer and the middle of a long hole stays bare; more just costs time,
// and past HEAL_FILL_MAX_LEVELS the blur is wider than the frame anyway.
//
// All of it is drawn on a surface the size of the brush's own box rather
// than the frame. The fill is read only where the mask is (the composite
// below mixes it at the mask's alpha, and nowhere else), so the pixels a
// wide blur would carry in from the frame's far corners are pixels nobody
// ever asks for — and at the zoom a repair is aimed at, lifting a wide
// blur over 3200x2400 instead of over the 500px around the repair is the
// difference between a second and a thirtieth of one (measured). The box
// is the mask's own, padded by three sigma of the widest level, so the
// blur at its edge still reads the rim it is made of rather than a clamp.
const widest = Math.max(...heal.map((s) => s.r * width));
const xs = heal.map((s) => s.x * width);
const ys = heal.map((s) => s.y * height);
const span = Math.hypot(Math.max(...xs) - Math.min(...xs), Math.max(...ys) - Math.min(...ys));
const levelCount = Math.min(
HEAL_FILL_MAX_LEVELS,
Math.max(2, 1 + Math.ceil(Math.log2(2 + span / 2 / widest)))
);
const pad = Math.max(64, Math.min(512, Math.round(widest * 3 + span * 0.6)));
const boxLeft = Math.max(0, Math.floor(Math.min(...xs) - widest - pad));
const boxTop = Math.max(0, Math.floor(Math.min(...ys) - widest - pad));
const boxRight = Math.min(width, Math.ceil(Math.max(...xs) + widest + pad));
const boxBottom = Math.min(height, Math.ceil(Math.max(...ys) + widest + pad));
const fillSurface = own(createSurface(boxRight - boxLeft, boxBottom - boxTop));
const fillCanvas = fillSurface.getCanvas();
// What the box names is drawn at the surface's own corner, 1:1 — the source
// rect is the frame's, the destination is the surface's, and the two are a
// whole number of pixels apart, which is the draw this Skia filters
// correctly. Drawn at the frame's own coordinates instead — both rects the
// box — the surface's left and top strips are left empty and its far side is
// clipped, and the composite that reads it back with
// `imageShaderChildAt(fillImage, boxLeft, boxTop)` then lands on the wrong
// pixels: the repaired patch takes the frame's colour from a boxLeft,boxTop
// up and to the left of where it is painted — a spot healed in the middle of
// the frame came out with the tone of the frame's top-left corner, a shade
// off on a smooth sky and the wrong colour outright on anything with shape.
const box = Skia.XYWHRect(boxLeft, boxTop, boxRight - boxLeft, boxBottom - boxTop);
const onto = Skia.XYWHRect(0, 0, boxRight - boxLeft, boxBottom - boxTop);
fillCanvas.clear(Skia.Color('transparent'));
if (cutImage) {
for (let k = levelCount; k >= 1; k--) {
const sigma = HEAL_FILL_BLUR * widest * 2 ** (k - 1);
const blurPaint = own(Skia.Paint());
blurPaint.setImageFilter(own(Skia.ImageFilter.MakeBlur(sigma, sigma, Skia.TileMode.Clamp, null)));
fillCanvas.drawImageRect(cutImage, box, onto, blurPaint);
}
// The cut itself, unblurred, on top: the pixels the brush did not touch
// are the photo exactly, not a blur of it.
fillCanvas.drawImageRect(cutImage, box, onto, own(Skia.Paint()));
}
// The other half of the fill, and the better one where it can be had: the
// pixels under the brush invented by MI-GAN from the pixels around them
// (engine/inpaint.ts) rather than averaged from the rim.
//
// One drag is one repair, and a drag that runs the length of the frame is
// solved in crops of its own extent rather than in one downscaled box that
// would invent the whole run at the resolution of its bounding rectangle.
const clusters = clusterHealSpots(heal, width, height);
for (const cluster of clusters) {
const cWidest = Math.max(...cluster.map((s) => s.r * width));
const cXs = cluster.map((s) => s.x * width);
const cYs = cluster.map((s) => s.y * height);
const mx0 = Math.min(...cXs) - cWidest;
const my0 = Math.min(...cYs) - cWidest;
const mx1 = Math.max(...cXs) + cWidest;
const my1 = Math.max(...cYs) + cWidest;
const cSpanW = mx1 - mx0;
const cSpanH = my1 - my0;
const cSpan = Math.max(cSpanW, cSpanH);
const margin = Math.max(
96,
Math.min(512, Math.round(cWidest * 3 + cSpan * 0.8))
);
const srcLeft = Math.max(boxLeft, Math.floor(mx0 - margin));
const srcTop = Math.max(boxTop, Math.floor(my0 - margin));
const srcRight = Math.min(boxRight, Math.ceil(mx1 + margin));
const srcBottom = Math.min(boxBottom, Math.ceil(my1 + margin));
const srcW = srcRight - srcLeft;
const srcH = srcBottom - srcTop;
if (srcW >= 8 && srcH >= 8) {
// The model's grid comes from the crop's share of the FRAME and not
// from the pixels the frame is rendered at, so the same repair is the
// same repair at every zoom and at the export's own size — see
// INPAINT_REF_LONG. It is also finer than it used to be wherever the
// stage is small: a crop a tenth of a 768px preview is 77 pixels, and
// 77 pixels of context is what the model used to be asked to invent
// from.
const { w: mw, h: mh } = inpaintCropSize(
(srcW / width) * INPAINT_REF_LONG,
(srcH / height) * INPAINT_REF_LONG
);
const cropSurface = own(createSurface(mw, mh));
if (cropSurface) {
// Neural inpainting (inpaint-web): read the clean, unblurred original pixels
// from snap so the model is fed crisp scene context without Gaussian pre-blurring.
cropSurface.getCanvas().drawImageRect(
snap,
Skia.XYWHRect(srcLeft, srcTop, srcW, srcH),
Skia.XYWHRect(0, 0, mw, mh),
own(Skia.Paint())
);
flush(cropSurface);
const cropPixels = cropSurface.getCanvas().readPixels(0, 0, {
width: mw,
height: mh,
colorType: Skia.ColorType.RGBA_8888,
alphaType: Skia.AlphaType.Unpremul,
colorSpace: Skia.ColorSpace.SRGB,
});
if (cropPixels) {
const kx = mw / srcW;
const ky = mh / srcH;
const hole = new Uint8Array(mw * mh);
for (const s of cluster) {
const cx = (s.x * width - srcLeft) * kx;
const cy = (s.y * height - srcTop) * ky;
const rad = s.r * width * kx;
const x0 = Math.max(0, Math.floor(cx - rad));
const x1 = Math.min(mw - 1, Math.ceil(cx + rad));
const y0 = Math.max(0, Math.floor(cy - rad));
const y1 = Math.min(mh - 1, Math.ceil(cy + rad));
for (let y = y0; y <= y1; y++) {
const dy = y - cy;
const row = y * mw;
for (let x = x0; x <= x1; x++) {
const dx = x - cx;
if (dx * dx + dy * dy <= rad * rad) hole[row + x] = 1;
}
}
}
// Compute normalized patchKey so preview zoom in/out hits cache with 0ms overhead
const patchKey = `${mw}x${mh}_` + cluster.map((s) => `${s.x.toFixed(4)},${s.y.toFixed(4)},${s.r.toFixed(4)}`).join(';');
(hole as { patchKey?: string }).patchKey = patchKey;
const patch = await inpaintCrop(cropPixels, mw, mh, hole).catch(() => null);
if (patch) {
const patchSurface = own(createSurface(mw, mh));
if (patchSurface && patchSurface.getCanvas().writePixels(patch, mw, mh, 0, 0)) {
flush(patchSurface);
const patchImage = own(patchSurface.makeImageSnapshot());
if (patchImage) {
// The model's pixels under the spot, and nowhere else at
// all. What shape they are laid through is the join between
// the model's invention and the photo's real texture, and it
// is not the mask's shape: the model is the repair wherever
// it has one, so it owns the whole spot and gives way only in
// the sliver the mask itself fades over (HEAL_PATCH_STOPS,
// and the numbers that put it there are in heal.ts).
const patchSurface = own(createSurface(srcW, srcH));
if (patchSurface) {
const patchCanvas = patchSurface.getCanvas();
patchCanvas.clear(Skia.Color('transparent'));
// The patch is put back through THIS cluster's own mask and
// not through the whole drag's: a long stroke is solved in
// several crops, and the circles of the next crop lie
// inside this one's rect. Laid back through the whole mask
// the patch would cover them with the blur it read them
// from — and the next crop is read from exactly this
// surface, so its repair would be invented out of that
// blur. Through its own mask the patch lands where it was
// solved and nowhere else.
for (const s of cluster) {
const cx = s.x * width - srcLeft;
const cy = s.y * height - srcTop;
const rad = s.r * width;
const shader = own(
Skia.Shader.MakeRadialGradient(
[cx, cy],
rad,
new Float32Array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0]),
HEAL_PATCH_STOPS,
Skia.TileMode.Clamp
)
);
const spotPaint = own(Skia.Paint());
spotPaint.setAntiAlias(true);
spotPaint.setShader(shader);
patchCanvas.drawCircle(cx, cy, rad, spotPaint);
}
const patchPaint = own(Skia.Paint());
patchPaint.setAntiAlias(true);
patchPaint.setBlendMode(Skia.BlendMode.SrcIn);
patchCanvas.drawImageRect(
patchImage,
Skia.XYWHRect(0, 0, mw, mh),
Skia.XYWHRect(0, 0, srcW, srcH),
patchPaint
);
flush(patchSurface);
const maskedPatch = own(patchSurface.makeImageSnapshot());
if (maskedPatch) {
fillCanvas.drawImageRect(
maskedPatch,
Skia.XYWHRect(0, 0, srcW, srcH),
Skia.XYWHRect(srcLeft - boxLeft, srcTop - boxTop, srcW, srcH),
own(Skia.Paint())
);
}
}
}
}
}
}
}
}
}
flush(fillSurface);
const fillImage = own(fillSurface.makeImageSnapshot());
// And the fill back under the brush, in one draw: the frame, the fill and
// the mask as its three children, with the mask the weight the frame and
// the fill are mixed at — so the rim is the only place the two meet, and
// there they already agree.
if (fillImage && snap && maskImage && fillEffect) {
const shader = own(
fillEffect.makeShaderWithChildren([], [
own(imageShaderChild(snap)),
own(imageShaderChildAt(fillImage, boxLeft, boxTop)),
own(imageShaderChild(maskImage)),
])
);
if (shader) {
canvas.clear(Skia.Color('transparent'));
const healPaint = own(Skia.Paint());
healPaint.setShader(shader);
canvas.drawRect(Skia.XYWHRect(0, 0, width, height), healPaint);
}
}
}
// 6f. MOSAIC — the FX tab's hiding brush (see mosaic.ts). It comes after
// HEAL because hiding is the last thing the photo's own pixels are asked
// for: a repair under a hidden patch is work nobody will ever see, and the
docker/frontend/src/engine/inpaint.ts
+133 -28
View File
@@ -116,12 +116,21 @@ function load(): Promise<Loaded> {
}
// Start fetching the model before the brush that needs it.
// Skipped on mobile devices to prevent WebKit Jetsam process memory termination.
export function preloadInpaint(): void {
if (isMobile()) return;
for (const file of [MODEL_URL, ...RUNTIME_FILES.map((f) => WASM_DIR + f), WASM_DIR + 'ort-wasm-simd-threaded.wasm']) {
void fetch(file).catch(() => {});
}
}
type ClearCallback = () => void;
const clearCallbacks: ClearCallback[] = [];
export function onClearInpaint(cb: ClearCallback): void {
clearCallbacks.push(cb);
}
/**
* Continuous Anisotropic Gradient & Texture Inpainting (Telea + Poisson boundary relaxation).
* Runs in pure TypeScript using <2MB RAM in ~15ms, guaranteeing 100% crash-free repair
@@ -150,9 +159,10 @@ export function fastLocalInpaint(
}
if (holeCount === 0) return out;
// 1. Distance transform from clean boundary
// 1. Distance transform from clean boundary into the hole
const dist = new Float32Array(total).fill(1e6);
const queue: number[] = [];
const boundaryDonors: number[] = [];
for (let y = 0; y < h; y++) {
const row = y * w;
@@ -160,6 +170,13 @@ export function fastLocalInpaint(
const idx = row + x;
if (!mask[idx]) {
dist[idx] = 0;
const up = y > 0 && mask[idx - w];
const down = y < h - 1 && mask[idx + w];
const left = x > 0 && mask[idx - 1];
const right = x < w - 1 && mask[idx + 1];
if (up || down || left || right) {
boundaryDonors.push(idx);
}
} else {
const up = y > 0 && !mask[idx - w];
const down = y < h - 1 && !mask[idx + w];
@@ -173,6 +190,8 @@ export function fastLocalInpaint(
}
}
if (boundaryDonors.length === 0) return out;
let head = 0;
while (head < queue.length) {
const idx = queue[head++];
@@ -199,6 +218,15 @@ export function fastLocalInpaint(
// Sort queue by distance: outer boundary to inner core
queue.sort((a, b) => dist[a] - dist[b]);
let sumHx = 0, sumHy = 0;
for (let i = 0; i < queue.length; i++) {
const idx = queue[i];
sumHx += idx % w;
sumHy += (idx / w) | 0;
}
const holeCx = sumHx / queue.length;
const holeCy = sumHy / queue.length;
// 2. Anisotropic gradient propagation along isophotes
const R = 7;
for (let i = 0; i < queue.length; i++) {
@@ -228,20 +256,17 @@ export function fastLocalInpaint(
const sIdx = srow + sx;
if (!mask[sIdx] || dist[sIdx] < dist[idx]) {
const dDist = Math.sqrt(d2);
// Directional weight along isophotes (tangent to normal)
const dir = Math.abs((dx * (-ny) + dy * nx) / dDist) * 0.7 + 0.3;
// Continuous Gaussian distance weight
const wSample = dir * Math.exp(-dDist / 3.0) / (1 + dist[sIdx] * 0.5);
const sp = sIdx * 4;
// First-order Taylor gradient extrapolation
let gx = 0, gy = 0;
if (sx > 0 && sx < w - 1) gx = (out[sp + 4] - out[sp - 4]) * 0.5;
if (sy > 0 && sy < h - 1) gy = (out[(srow + w + sx) * 4] - out[(srow - w + sx) * 4]) * 0.5;
const rExt = out[sp] + gx * (-dx) * 0.5;
const gExt = out[sp + 1] + gx * (-dx) * 0.5;
const bExt = out[sp + 2] + gx * (-dx) * 0.5;
const rExt = out[sp] + gx * (-dx) * 0.3;
const gExt = out[sp + 1] + gy * (-dy) * 0.3;
const bExt = out[sp + 2] + (gx + gy) * 0.5 * (-dx - dy) * 0.15;
sumR += Math.max(0, Math.min(255, rExt)) * wSample;
sumG += Math.max(0, Math.min(255, gExt)) * wSample;
@@ -256,11 +281,28 @@ export function fastLocalInpaint(
out[p] = Math.round(sumR / sumW);
out[p + 1] = Math.round(sumG / sumW);
out[p + 2] = Math.round(sumB / sumW);
} else {
let bestD2 = 1e9;
let bestP = boundaryDonors[0] * 4;
const bStep = Math.max(1, Math.floor(boundaryDonors.length / 32));
for (let k = 0; k < boundaryDonors.length; k += bStep) {
const bIdx = boundaryDonors[k];
const bx = bIdx % w;
const by = (bIdx / w) | 0;
const bd2 = (px - bx) ** 2 + (py - by) ** 2;
if (bd2 < bestD2) {
bestD2 = bd2;
bestP = bIdx * 4;
}
}
out[p] = out[bestP];
out[p + 1] = out[bestP + 1];
out[p + 2] = out[bestP + 2];
}
}
// 3. Multi-scale Poisson relaxation
for (let pass = 0; pass < 5; pass++) {
for (let pass = 0; pass < 4; pass++) {
for (let i = 0; i < queue.length; i++) {
const idx = queue[i];
const px = idx % w;
@@ -275,37 +317,84 @@ export function fastLocalInpaint(
for (let c = 0; c < 3; c++) {
const avg = (out[l + c] + out[r + c] + out[u + c] + out[d + c]) * 0.25;
out[p + c] = Math.round(out[p + c] * 0.3 + avg * 0.7);
out[p + c] = Math.round(out[p + c] * 0.35 + avg * 0.65);
}
}
}
// 4. Photorealistic texture & grain matching
let varSum = 0, varCount = 0;
for (let y = 1; y < h - 1; y += 2) {
const row = y * w;
for (let x = 1; x < w - 1; x += 2) {
const idx = row + x;
if (!mask[idx]) {
const p = idx * 4;
const diff = Math.abs(out[p] - out[p + 4]) + Math.abs(out[p] - out[(y + 1) * w * 4 + x * 4]);
varSum += diff;
varCount++;
}
// 4. Authentic Exemplar Texture & Grain Transfer
const bCount = boundaryDonors.length;
const bStep = Math.max(1, Math.floor(bCount / 64));
let noiseVarSum = 0;
let noiseVarCount = 0;
for (let i = 0; i < bCount; i += bStep) {
const bIdx = boundaryDonors[i];
const bx = bIdx % w;
const by = (bIdx / w) | 0;
if (bx > 0 && bx < w - 1 && by > 0 && by < h - 1) {
const bp = bIdx * 4;
const avgSurrR = (rgba[bp - 4] + rgba[bp + 4] + rgba[(by - 1) * w * 4 + bx * 4] + rgba[(by + 1) * w * 4 + bx * 4]) * 0.25;
const avgSurrG = (rgba[bp - 3] + rgba[bp + 5] + rgba[(by - 1) * w * 4 + bx * 4 + 1] + rgba[(by + 1) * w * 4 + bx * 4 + 1]) * 0.25;
const avgSurrB = (rgba[bp - 2] + rgba[bp + 6] + rgba[(by - 1) * w * 4 + bx * 4 + 2] + rgba[(by + 1) * w * 4 + bx * 4 + 2]) * 0.25;
const diffR = rgba[bp] - avgSurrR;
const diffG = rgba[bp + 1] - avgSurrG;
const diffB = rgba[bp + 2] - avgSurrB;
noiseVarSum += Math.abs(diffR) + Math.abs(diffG) + Math.abs(diffB);
noiseVarCount++;
}
}
const grainAmp = varCount > 0 ? Math.min(4.5, (varSum / varCount) * 0.4) : 1.5;
const avgTextureEnergy = noiseVarCount > 0 ? (noiseVarSum / noiseVarCount) / 3 : 1.5;
let seed = 98765;
const rand = () => { seed = (seed * 16807) % 2147483647; return (seed / 2147483647) - 0.5; };
let seed = 1234567;
const rand = () => {
seed = (seed * 16807) % 2147483647;
return (seed / 2147483647) - 0.5;
};
for (let i = 0; i < queue.length; i++) {
const idx = queue[i];
const px = idx % w;
const py = (idx / w) | 0;
const mx = Math.max(0, Math.min(w - 1, Math.round(2 * holeCx - px)));
const my = Math.max(0, Math.min(h - 1, Math.round(2 * holeCy - py)));
const mIdx = my * w + mx;
let donorIdx = -1;
if (!mask[mIdx]) {
donorIdx = mIdx;
} else {
let bestDist = 1e9;
for (let k = 0; k < bCount; k += bStep) {
const bp = boundaryDonors[k];
const bbx = bp % w;
const bby = (bp / w) | 0;
const d = (px - bbx) ** 2 + (py - bby) ** 2;
if (d < bestDist) {
bestDist = d;
donorIdx = bp;
}
}
}
const p = idx * 4;
const noise = rand() * grainAmp;
out[p] = Math.max(0, Math.min(255, Math.round(out[p] + noise)));
out[p + 1] = Math.max(0, Math.min(255, Math.round(out[p + 1] + noise)));
out[p + 2] = Math.max(0, Math.min(255, Math.round(out[p + 2] + noise)));
if (donorIdx >= 0) {
const dp = donorIdx * 4;
const dx = donorIdx % w;
const dy = (donorIdx / w) | 0;
let highR = 0, highG = 0, highB = 0;
if (dx > 0 && dx < w - 1 && dy > 0 && dy < h - 1) {
highR = rgba[dp] - (rgba[dp - 4] + rgba[dp + 4] + rgba[(dy - 1) * w * 4 + dx * 4] + rgba[(dy + 1) * w * 4 + dx * 4]) * 0.25;
highG = rgba[dp + 1] - (rgba[dp - 3] + rgba[dp + 5] + rgba[(dy - 1) * w * 4 + dx * 4 + 1] + rgba[(dy + 1) * w * 4 + dx * 4 + 1]) * 0.25;
highB = rgba[dp + 2] - (rgba[dp - 2] + rgba[dp + 6] + rgba[(dy - 1) * w * 4 + dx * 4 + 2] + rgba[(dy + 1) * w * 4 + dx * 4 + 2]) * 0.25;
}
const grain = rand() * Math.min(3.5, avgTextureEnergy * 0.5);
out[p] = Math.max(0, Math.min(255, Math.round(out[p] + highR * 0.75 + grain)));
out[p + 1] = Math.max(0, Math.min(255, Math.round(out[p + 1] + highG * 0.75 + grain)));
out[p + 2] = Math.max(0, Math.min(255, Math.round(out[p + 2] + highB * 0.75 + grain)));
}
out[p + 3] = 255;
}
return out;
@@ -362,6 +451,9 @@ let inFlightInpaint: Promise<Uint8Array | null> | null = null;
export function clearInpaintCache(): void {
patchCache.clear();
for (const cb of clearCallbacks) {
try { cb(); } catch {}
}
}
export async function releaseInpaint(): Promise<void> {
@@ -415,6 +507,19 @@ export async function inpaintCrop(
return patchCache.get(key)!.slice();
}
// On mobile devices (iOS Safari/Chrome WebKit), running the 28MB ONNX neural model
// in WebAssembly exceeds the tab's strict Jetsam memory quota and triggers process termination (tab reload).
// Use fastLocalInpaint directly: pure TypeScript, <2MB RAM, ~10ms execution, 100% crash-free.
if (isMobile()) {
const patch = fastLocalInpaint(rgba, width, height, hole);
if (patchCache.size > 50) {
const first = patchCache.keys().next().value;
if (first) patchCache.delete(first);
}
patchCache.set(key, patch.slice());
return patch;
}
// Single-flight lock: serialize inference executions to prevent memory spikes
if (inFlightInpaint) {
try {