3dtours/RecipesCam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(heal): apply HEAL directly to base image before exposure grading and use fast texture synthesis on mobile to prevent iOS reload

Browse Source
This commit is contained in:
3dtours
2026-10-16 12:04:18 +07:00
parent d579a67ed9
commit f96a60f416
3 changed files with 514 additions and 447 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/frontend/src/engine/exportEngine.ts
+380 -418
View File
@@ -56,7 +56,7 @@ import {
HEAL_PATCH_STOPS,
readHeal,
} from '../../shared/utils/heal';
import { inpaintCrop, inpaintCropSize, INPAINT_REF_LONG } from './inpaint';
import { inpaintCrop, inpaintCropSize, INPAINT_REF_LONG, onClearInpaint } from './inpaint';
import { mosaicSkSL, mosaicUniforms, readMosaic } from '../../shared/utils/mosaic';
import { gradientMaskSkSL, maskUniforms, readMasks, masksHaveSpatial } from '../../shared/utils/gradientMask';
import {
@@ -873,6 +873,370 @@ function loadOldFilm(assets: RenderAssets) {
return oldFilmImage;
}
function clusterHealSpots(
spots: { x: number; y: number; r: number; stroke?: number }[],
width: number,
height: number
): { x: number; y: number; r: number; stroke?: number }[][] {
if (spots.length <= 1) return spots.length ? [spots] : [];
const n = spots.length;
const adj: number[][] = Array.from({ length: n }, () => []);
for (let i = 0; i < n; i++) {
const si = spots[i];
const xi = si.x * width;
const yi = si.y * height;
const ri = si.r * width;
for (let j = i + 1; j < n; j++) {
const sj = spots[j];
if (si.stroke !== sj.stroke) continue;
const xj = sj.x * width;
const yj = sj.y * height;
const rj = sj.r * width;
const maxDist = (ri + rj) * 1.5;
const distSq = (xi - xj) ** 2 + (yi - yj) ** 2;
if (distSq <= maxDist * maxDist) {
adj[i].push(j);
adj[j].push(i);
}
}
}
const isMob = typeof navigator !== 'undefined' && /iPad|iPhone|iPod|Android|Mobile/i.test(navigator.userAgent || '');
const maxSpan = isMob ? 600 : 1024;
const visited = new Uint8Array(n);
const clusters: { x: number; y: number; r: number; stroke?: number }[][] = [];
for (let i = 0; i < n; i++) {
if (visited[i]) continue;
visited[i] = 1;
const cluster: { x: number; y: number; r: number; stroke?: number }[] = [spots[i]];
let minX = spots[i].x * width;
let maxX = minX;
let minY = spots[i].y * height;
let maxY = minY;
const queue = [i];
while (queue.length > 0) {
const curr = queue.shift()!;
for (const neighbor of adj[curr]) {
if (!visited[neighbor]) {
const nx = spots[neighbor].x * width;
const ny = spots[neighbor].y * height;
const newMinX = Math.min(minX, nx);
const newMaxX = Math.max(maxX, nx);
const newMinY = Math.min(minY, ny);
const newMaxY = Math.max(maxY, ny);
if (newMaxX - newMinX <= maxSpan && newMaxY - newMinY <= maxSpan) {
visited[neighbor] = 1;
cluster.push(spots[neighbor]);
minX = newMinX;
maxX = newMaxX;
minY = newMinY;
maxY = newMaxY;
queue.push(neighbor);
}
}
}
}
clusters.push(cluster);
}
return clusters;
}
interface HealedBaseCache {
baseImage: any;
healKey: string;
width: number;
height: number;
healedSnapshot: any;
}
let cachedHealedBase: HealedBaseCache | null = null;
export function clearHealBaseCache(): void {
if (cachedHealedBase?.healedSnapshot) {
try {
cachedHealedBase.healedSnapshot.dispose?.();
} catch {}
}
cachedHealedBase = null;
}
onClearInpaint(clearHealBaseCache);
async function applyHealToBase(
baseImage: any,
healSpots: { x: number; y: number; r: number; stroke?: number }[],
width: number,
height: number
): Promise<any> {
const healKey = healSpots
.map((s) => `${s.x.toFixed(4)},${s.y.toFixed(4)},${s.r.toFixed(4)},${s.stroke ?? 0}`)
.join(';');
if (
cachedHealedBase &&
cachedHealedBase.baseImage === baseImage &&
cachedHealedBase.healKey === healKey &&
cachedHealedBase.width === width &&
cachedHealedBase.height === height
) {
return cachedHealedBase.healedSnapshot;
}
let curImage = baseImage;
const tempOwned: any[] = [];
const tempOwn = <T,>(obj: T | null | undefined): T => {
if (obj) tempOwned.push(obj);
return obj as T;
};
try {
const { healCutEffect: cutEffect, healFillEffect: fillEffect } = effects();
const clusters = clusterHealSpots(healSpots, width, height);
for (const heal of clusters) {
const snap = curImage;
const maskSurface = tempOwn(createSurface(width, height));
if (!maskSurface) continue;
const maskCanvas = maskSurface.getCanvas();
maskCanvas.clear(Skia.Color('transparent'));
for (const s of heal) {
const cx = s.x * width;
const cy = s.y * height;
const rad = s.r * width;
const shader = tempOwn(
Skia.Shader.MakeRadialGradient(
[cx, cy],
rad,
new Float32Array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0]),
HEAL_MASK_STOPS,
Skia.TileMode.Clamp
)
);
const paint = tempOwn(Skia.Paint());
paint.setAntiAlias(true);
paint.setShader(shader);
maskCanvas.drawCircle(cx, cy, rad, paint);
}
flush(maskSurface);
const maskImage = tempOwn(maskSurface.makeImageSnapshot());
const cutSurface = tempOwn(createSurface(width, height));
if (snap && maskImage && cutEffect && cutSurface) {
const cutShader = tempOwn(
cutEffect.makeShaderWithChildren([], [tempOwn(imageShaderChild(snap)), tempOwn(imageShaderChild(maskImage))])
);
if (cutShader) {
const cutCanvas = cutSurface.getCanvas();
cutCanvas.clear(Skia.Color('transparent'));
const cutPaint = tempOwn(Skia.Paint());
cutPaint.setShader(cutShader);
cutCanvas.drawRect(Skia.XYWHRect(0, 0, width, height), cutPaint);
}
}
flush(cutSurface);
const cutImage = tempOwn(cutSurface.makeImageSnapshot());
const widest = Math.max(...heal.map((s) => s.r * width));
const xs = heal.map((s) => s.x * width);
const ys = heal.map((s) => s.y * height);
const span = Math.hypot(Math.max(...xs) - Math.min(...xs), Math.max(...ys) - Math.min(...ys));
const levelCount = Math.min(
HEAL_FILL_MAX_LEVELS,
Math.max(2, 1 + Math.ceil(Math.log2(2 + span / 2 / widest)))
);
const pad = Math.max(64, Math.min(512, Math.round(widest * 3 + span * 0.6)));
const boxLeft = Math.max(0, Math.floor(Math.min(...xs) - widest - pad));
const boxTop = Math.max(0, Math.floor(Math.min(...ys) - widest - pad));
const boxRight = Math.min(width, Math.ceil(Math.max(...xs) + widest + pad));
const boxBottom = Math.min(height, Math.ceil(Math.max(...ys) + widest + pad));
const fillSurface = tempOwn(createSurface(boxRight - boxLeft, boxBottom - boxTop));
if (!fillSurface) continue;
const fillCanvas = fillSurface.getCanvas();
const box = Skia.XYWHRect(boxLeft, boxTop, boxRight - boxLeft, boxBottom - boxTop);
const onto = Skia.XYWHRect(0, 0, boxRight - boxLeft, boxBottom - boxTop);
fillCanvas.clear(Skia.Color('transparent'));
if (cutImage) {
for (let k = levelCount; k >= 1; k--) {
const sigma = HEAL_FILL_BLUR * widest * 2 ** (k - 1);
const blurPaint = tempOwn(Skia.Paint());
blurPaint.setImageFilter(tempOwn(Skia.ImageFilter.MakeBlur(sigma, sigma, Skia.TileMode.Clamp, null)));
fillCanvas.drawImageRect(cutImage, box, onto, blurPaint);
}
fillCanvas.drawImageRect(cutImage, box, onto, tempOwn(Skia.Paint()));
}
const subClusters = clusterHealSpots(heal, width, height);
for (const cluster of subClusters) {
const cWidest = Math.max(...cluster.map((s) => s.r * width));
const cXs = cluster.map((s) => s.x * width);
const cYs = cluster.map((s) => s.y * height);
const mx0 = Math.min(...cXs) - cWidest;
const my0 = Math.min(...cYs) - cWidest;
const mx1 = Math.max(...cXs) + cWidest;
const my1 = Math.max(...cYs) + cWidest;
const cSpanW = mx1 - mx0;
const cSpanH = my1 - my0;
const cSpan = Math.max(cSpanW, cSpanH);
const margin = Math.max(96, Math.min(512, Math.round(cWidest * 3 + cSpan * 0.8)));
const srcLeft = Math.max(boxLeft, Math.floor(mx0 - margin));
const srcTop = Math.max(boxTop, Math.floor(my0 - margin));
const srcRight = Math.min(boxRight, Math.ceil(mx1 + margin));
const srcBottom = Math.min(boxBottom, Math.ceil(my1 + margin));
const srcW = srcRight - srcLeft;
const srcH = srcBottom - srcTop;
if (srcW >= 8 && srcH >= 8) {
const { w: mw, h: mh } = inpaintCropSize(
(srcW / width) * INPAINT_REF_LONG,
(srcH / height) * INPAINT_REF_LONG
);
const cropSurface = tempOwn(createSurface(mw, mh));
if (cropSurface) {
cropSurface.getCanvas().drawImageRect(
snap,
Skia.XYWHRect(srcLeft, srcTop, srcW, srcH),
Skia.XYWHRect(0, 0, mw, mh),
tempOwn(Skia.Paint())
);
flush(cropSurface);
const cropPixels = cropSurface.getCanvas().readPixels(0, 0, {
width: mw,
height: mh,
colorType: Skia.ColorType.RGBA_8888,
alphaType: Skia.AlphaType.Unpremul,
colorSpace: Skia.ColorSpace.SRGB,
});
if (cropPixels) {
const kx = mw / srcW;
const ky = mh / srcH;
const hole = new Uint8Array(mw * mh);
for (const s of cluster) {
const cx = (s.x * width - srcLeft) * kx;
const cy = (s.y * height - srcTop) * ky;
const rad = s.r * width * kx;
const x0 = Math.max(0, Math.floor(cx - rad));
const x1 = Math.min(mw - 1, Math.ceil(cx + rad));
const y0 = Math.max(0, Math.floor(cy - rad));
const y1 = Math.min(mh - 1, Math.ceil(cy + rad));
for (let y = y0; y <= y1; y++) {
const dy = y - cy;
const row = y * mw;
for (let x = x0; x <= x1; x++) {
const dx = x - cx;
if (dx * dx + dy * dy <= rad * rad) hole[row + x] = 1;
}
}
}
const patchKey = `${mw}x${mh}_` + cluster.map((s) => `${s.x.toFixed(4)},${s.y.toFixed(4)},${s.r.toFixed(4)}`).join(';');
(hole as { patchKey?: string }).patchKey = patchKey;
const patch = await inpaintCrop(cropPixels, mw, mh, hole).catch(() => null);
if (patch) {
const patchSurface = tempOwn(createSurface(mw, mh));
if (patchSurface && patchSurface.getCanvas().writePixels(patch, mw, mh, 0, 0)) {
flush(patchSurface);
const patchImage = tempOwn(patchSurface.makeImageSnapshot());
if (patchImage) {
const patchMaskSurface = tempOwn(createSurface(srcW, srcH));
if (patchMaskSurface) {
const patchCanvas = patchMaskSurface.getCanvas();
patchCanvas.clear(Skia.Color('transparent'));
for (const s of cluster) {
const cx = s.x * width - srcLeft;
const cy = s.y * height - srcTop;
const rad = s.r * width;
const shader = tempOwn(
Skia.Shader.MakeRadialGradient(
[cx, cy],
rad,
new Float32Array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0]),
HEAL_PATCH_STOPS,
Skia.TileMode.Clamp
)
);
const spotPaint = tempOwn(Skia.Paint());
spotPaint.setAntiAlias(true);
spotPaint.setShader(shader);
patchCanvas.drawCircle(cx, cy, rad, spotPaint);
}
const patchPaint = tempOwn(Skia.Paint());
patchPaint.setAntiAlias(true);
patchPaint.setBlendMode(Skia.BlendMode.SrcIn);
patchCanvas.drawImageRect(
patchImage,
Skia.XYWHRect(0, 0, mw, mh),
Skia.XYWHRect(0, 0, srcW, srcH),
patchPaint
);
flush(patchMaskSurface);
const maskedPatch = tempOwn(patchMaskSurface.makeImageSnapshot());
if (maskedPatch) {
fillCanvas.drawImageRect(
maskedPatch,
Skia.XYWHRect(0, 0, srcW, srcH),
Skia.XYWHRect(srcLeft - boxLeft, srcTop - boxTop, srcW, srcH),
tempOwn(Skia.Paint())
);
}
}
}
}
}
}
}
}
}
flush(fillSurface);
const fillImage = tempOwn(fillSurface.makeImageSnapshot());
if (fillImage && snap && maskImage && fillEffect) {
const shader = tempOwn(
fillEffect.makeShaderWithChildren([], [
tempOwn(imageShaderChild(snap)),
tempOwn(imageShaderChildAt(fillImage, boxLeft, boxTop)),
tempOwn(imageShaderChild(maskImage)),
])
);
if (shader) {
const clusterSurface = tempOwn(createSurface(width, height));
if (clusterSurface) {
const clusterCanvas = clusterSurface.getCanvas();
clusterCanvas.clear(Skia.Color('transparent'));
const healPaint = tempOwn(Skia.Paint());
healPaint.setShader(shader);
clusterCanvas.drawRect(Skia.XYWHRect(0, 0, width, height), healPaint);
flush(clusterSurface);
const clusterSnap = clusterSurface.makeImageSnapshot();
if (clusterSnap) {
curImage = clusterSnap;
tempOwn(clusterSnap);
}
}
}
}
}
if (cachedHealedBase?.healedSnapshot && cachedHealedBase.healedSnapshot !== curImage) {
try { cachedHealedBase.healedSnapshot.dispose?.(); } catch {}
}
cachedHealedBase = {
baseImage,
healKey,
width,
height,
healedSnapshot: curImage,
};
return curImage;
} finally {
for (const obj of tempOwned) {
if (obj !== curImage) {
try { obj.dispose?.(); } catch {}
}
}
}
}
export async function renderPhoto(input: RenderInput): Promise<RenderResult | null> {
const { recipe, frameId, options } = input;
const owned: SkDisposable[] = [];
@@ -957,6 +1321,21 @@ export async function renderPhoto(input: RenderInput): Promise<RenderResult | nu
const adjustments = recipe.adjustments;
// 2b. Base photo HEAL: applied directly to skImage before color grading and exposure.
// In professional RAW engines (Lightroom, Camera Raw), spot healing operates on
// base pixels so that exposure and tone adjustments grade the healed area and
// surrounding pixels uniformly in exact unison, preventing patches from standing
// out when brightness is adjusted.
const healSpots = readHeal(adjustments.heal);
if (healSpots.length > 0) {
const healed = await applyHealToBase(skImage, healSpots, width, height);
if (healed && healed !== skImage) {
own(healed);
release(owned, skImage);
skImage = healed;
}
}
// 3. Colour matrix, then the exposure pass (+EV unless the camera already
// biased the shot). The matrix grades into its OWN image first, because a
// SkPaint runs its shader before its colourFilter: with both on one paint the
@@ -1383,423 +1762,6 @@ export async function renderPhoto(input: RenderInput): Promise<RenderResult | nu
});
}
function clusterHealSpots(
spots: { x: number; y: number; r: number; stroke?: number }[],
width: number,
height: number
): { x: number; y: number; r: number }[][] {
if (spots.length <= 1) return spots.length ? [spots] : [];
const n = spots.length;
const adj: number[][] = Array.from({ length: n }, () => []);
for (let i = 0; i < n; i++) {
const si = spots[i];
const xi = si.x * width;
const yi = si.y * height;
const ri = si.r * width;
for (let j = i + 1; j < n; j++) {
const sj = spots[j];
// Two drags are two repairs even where their circles touch. A stroke is
// HEALed when the user lets go of it, and what the solve below reads is
// the frame the earlier ones have already left — so a stroke next to a
// healed one is drawn from a photo that no longer holds the first, and
// never carries its detail into the second. One cluster spanning both
// would be a single crop of the pair, solved from the ground around the
// two: the detail of one repair coming back inside the other. Spots
// written before the brush numbered its drags have none, and stay
// grouped by distance, the reading they always got.
if (si.stroke !== sj.stroke) continue;
const xj = sj.x * width;
const yj = sj.y * height;
const rj = sj.r * width;
const maxDist = (ri + rj) * 1.5;
const distSq = (xi - xj) ** 2 + (yi - yj) ** 2;
if (distSq <= maxDist * maxDist) {
adj[i].push(j);
adj[j].push(i);
}
}
}
// Dimension of a cluster's bounding box. Allows unified processing of connected
// brush strokes in a single fast pass instead of slow fragmented passes.
const isMob = typeof navigator !== 'undefined' && /iPad|iPhone|iPod|Android|Mobile/i.test(navigator.userAgent || '');
const maxSpan = isMob ? 600 : 1024;
const visited = new Uint8Array(n);
const clusters: { x: number; y: number; r: number }[][] = [];
for (let i = 0; i < n; i++) {
if (visited[i]) continue;
visited[i] = 1;
const cluster: { x: number; y: number; r: number }[] = [spots[i]];
let minX = spots[i].x * width;
let maxX = minX;
let minY = spots[i].y * height;
let maxY = minY;
const queue = [i];
while (queue.length > 0) {
const curr = queue.shift()!;
for (const neighbor of adj[curr]) {
if (!visited[neighbor]) {
const nx = spots[neighbor].x * width;
const ny = spots[neighbor].y * height;
const newMinX = Math.min(minX, nx);
const newMaxX = Math.max(maxX, nx);
const newMinY = Math.min(minY, ny);
const newMaxY = Math.max(maxY, ny);
if (newMaxX - newMinX <= maxSpan && newMaxY - newMinY <= maxSpan) {
visited[neighbor] = 1;
cluster.push(spots[neighbor]);
minX = newMinX;
maxX = newMaxX;
minY = newMinY;
maxY = newMaxY;
queue.push(neighbor);
}
}
}
}
clusters.push(cluster);
}
return clusters;
}
// 6e. HEAL — the FX tab's dust brush (see heal.ts). It is the last thing
// that touches the photo's own pixels, and deliberately so: the fill is made
// of the pixels the pipeline has already built, so a repair matches the
// grade, the blur and the grain around it by construction instead of being a
// second render of the same look that has to agree. The card, the wall and
// the watermarks are drawn after it, so a healed spot still carries the
// frame's own edge.
//
// Three moves, in this order. The brush's circles become ONE mask — a union,
// which is what makes a stroke a single shape instead of a row of circles.
// The mask is cut out of the frame, leaving a hole with the photo's own
// pixels all around its rim. And the hole is filled from that rim, by a
// blur of the frame, so nothing is copied from anywhere: what was under the
// brush is the light of what borders it.
// ONE DRAG AT A TIME, in the order the user made them, each solved against
// the frame the ones before it have already left. This is what "HEALed"
// means: the repair a stroke left behind is part of the photo from then on,
// and the drag laid next to it is invented from a frame that no longer holds
// what the first drag took away. Solved as one list instead — which is what
// this did, and what the whole of it below still does, once per drag — the
// second brush reaches back into the first: the crop around the new stroke
// covers the old ones, so they are re-derived from a frame that still had
// their dust in it, and a repair the user has already looked at and accepted
// moves under them the moment they brush beside it.
//
// The cost is one pass of the block below per drag rather than one for the
// photo: on a photo with a handful of repairs that is a handful of masks and
// composites, and the box each fill is built on is still the stroke's own,
// so nothing here is proportional to the frame but the mask and the one draw
// that puts the repair back. ponytail: a photo with dozens of repairs pays
// for all of them on every preview; the ceiling is a cache of the frame as
// it stands after each drag, keyed by the list up to that drag, and it is
// worth its bookkeeping only if the preview is ever felt to lag.
const healSpots = readHeal(adjustments.heal);
for (const heal of healSpots.length ? clusterHealSpots(healSpots, width, height) : []) {
const { healCutEffect: cutEffect, healFillEffect: fillEffect } = effects();
flush(surface);
const snap = own(surface.makeImageSnapshot());
// The mask, at the frame's own resolution: one radial gradient per spot,
// solid out to HEAL_FEATHER of its radius and clear at its edge, each laid
// over the last — so two spots of one stroke are one shape with no seam
// between them, and the mask's own border is a fade rather than an outline
// the fill would have to match exactly.
const maskSurface = own(createSurface(width, height));
const maskCanvas = maskSurface.getCanvas();
maskCanvas.clear(Skia.Color('transparent'));
for (const s of heal) {
const cx = s.x * width;
const cy = s.y * height;
const rad = s.r * width;
const shader = own(
Skia.Shader.MakeRadialGradient(
[cx, cy],
rad,
new Float32Array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0]),
HEAL_MASK_STOPS,
Skia.TileMode.Clamp
)
);
const paint = own(Skia.Paint());
paint.setAntiAlias(true);
paint.setShader(shader);
maskCanvas.drawCircle(cx, cy, rad, paint);
}
flush(maskSurface);
const maskImage = own(maskSurface.makeImageSnapshot());
// The frame with the mask taken out of it: the hole the fill works on.
const cutSurface = own(createSurface(width, height));
if (snap && maskImage && cutEffect) {
const cutShader = own(
cutEffect.makeShaderWithChildren([], [own(imageShaderChild(snap)), own(imageShaderChild(maskImage))])
);
if (cutShader) {
const cutCanvas = cutSurface.getCanvas();
cutCanvas.clear(Skia.Color('transparent'));
const cutPaint = own(Skia.Paint());
cutPaint.setShader(cutShader);
cutCanvas.drawRect(Skia.XYWHRect(0, 0, width, height), cutPaint);
}
}
flush(cutSurface);
const cutImage = own(cutSurface.makeImageSnapshot());
// The fill: the cut frame blurred at a few widths, widest first, each
// finer one drawn over it, the untouched cut on top. A blur weighs every
// pixel by how much of the frame's light reaches it, and the hole has none
// of its own left to weigh — so the hole comes back as the light of its
// rim, carried in with no seam and nothing copied from anywhere. Widest
// first means the widest blur only has a say where every finer one has
// nothing (the middle of a hole deeper than a brush), and the untouched
// cut wins wherever the photo is still there, so the rim is the rim.
//
// Blurs and not a pyramid of halvings, on purpose. This Skia gets the
// premultiply wrong on a snapshot used as a SCALED draw source — its rgb
// comes back with the alpha multiplied in a second time — and a pyramid is
// built of exactly those halvings; a partly transparent rim then came out
// a wash of grey, which is what the heal was painting over the photo.
// Every draw here is 1:1, which filters correctly (measured, see
// heal-check.mjs).
//
// How many levels: the finest is HEAL_FILL_BLUR brushes wide and each one
// is twice the last, so n of them reach HEAL_FILL_BLUR * 2^(n-1) radii —
// enough to span half the stroke's own diagonal from either end of it.
// Fewer and the middle of a long hole stays bare; more just costs time,
// and past HEAL_FILL_MAX_LEVELS the blur is wider than the frame anyway.
//
// All of it is drawn on a surface the size of the brush's own box rather
// than the frame. The fill is read only where the mask is (the composite
// below mixes it at the mask's alpha, and nowhere else), so the pixels a
// wide blur would carry in from the frame's far corners are pixels nobody
// ever asks for — and at the zoom a repair is aimed at, lifting a wide
// blur over 3200x2400 instead of over the 500px around the repair is the
// difference between a second and a thirtieth of one (measured). The box
// is the mask's own, padded by three sigma of the widest level, so the
// blur at its edge still reads the rim it is made of rather than a clamp.
const widest = Math.max(...heal.map((s) => s.r * width));
const xs = heal.map((s) => s.x * width);
const ys = heal.map((s) => s.y * height);
const span = Math.hypot(Math.max(...xs) - Math.min(...xs), Math.max(...ys) - Math.min(...ys));
const levelCount = Math.min(
HEAL_FILL_MAX_LEVELS,
Math.max(2, 1 + Math.ceil(Math.log2(2 + span / 2 / widest)))
);
const pad = Math.max(64, Math.min(512, Math.round(widest * 3 + span * 0.6)));
const boxLeft = Math.max(0, Math.floor(Math.min(...xs) - widest - pad));
const boxTop = Math.max(0, Math.floor(Math.min(...ys) - widest - pad));
const boxRight = Math.min(width, Math.ceil(Math.max(...xs) + widest + pad));
const boxBottom = Math.min(height, Math.ceil(Math.max(...ys) + widest + pad));
const fillSurface = own(createSurface(boxRight - boxLeft, boxBottom - boxTop));
const fillCanvas = fillSurface.getCanvas();
// What the box names is drawn at the surface's own corner, 1:1 — the source
// rect is the frame's, the destination is the surface's, and the two are a
// whole number of pixels apart, which is the draw this Skia filters
// correctly. Drawn at the frame's own coordinates instead — both rects the
// box — the surface's left and top strips are left empty and its far side is
// clipped, and the composite that reads it back with
// `imageShaderChildAt(fillImage, boxLeft, boxTop)` then lands on the wrong
// pixels: the repaired patch takes the frame's colour from a boxLeft,boxTop
// up and to the left of where it is painted — a spot healed in the middle of
// the frame came out with the tone of the frame's top-left corner, a shade
// off on a smooth sky and the wrong colour outright on anything with shape.
const box = Skia.XYWHRect(boxLeft, boxTop, boxRight - boxLeft, boxBottom - boxTop);
const onto = Skia.XYWHRect(0, 0, boxRight - boxLeft, boxBottom - boxTop);
fillCanvas.clear(Skia.Color('transparent'));
if (cutImage) {
for (let k = levelCount; k >= 1; k--) {
const sigma = HEAL_FILL_BLUR * widest * 2 ** (k - 1);
const blurPaint = own(Skia.Paint());
blurPaint.setImageFilter(own(Skia.ImageFilter.MakeBlur(sigma, sigma, Skia.TileMode.Clamp, null)));
fillCanvas.drawImageRect(cutImage, box, onto, blurPaint);
}
// The cut itself, unblurred, on top: the pixels the brush did not touch
// are the photo exactly, not a blur of it.
fillCanvas.drawImageRect(cutImage, box, onto, own(Skia.Paint()));
}
// The other half of the fill, and the better one where it can be had: the
// pixels under the brush invented by MI-GAN from the pixels around them
// (engine/inpaint.ts) rather than averaged from the rim.
//
// One drag is one repair, and a drag that runs the length of the frame is
// solved in crops of its own extent rather than in one downscaled box that
// would invent the whole run at the resolution of its bounding rectangle.
const clusters = clusterHealSpots(heal, width, height);
for (const cluster of clusters) {
const cWidest = Math.max(...cluster.map((s) => s.r * width));
const cXs = cluster.map((s) => s.x * width);
const cYs = cluster.map((s) => s.y * height);
const mx0 = Math.min(...cXs) - cWidest;
const my0 = Math.min(...cYs) - cWidest;
const mx1 = Math.max(...cXs) + cWidest;
const my1 = Math.max(...cYs) + cWidest;
const cSpanW = mx1 - mx0;
const cSpanH = my1 - my0;
const cSpan = Math.max(cSpanW, cSpanH);
const margin = Math.max(
96,
Math.min(512, Math.round(cWidest * 3 + cSpan * 0.8))
);
const srcLeft = Math.max(boxLeft, Math.floor(mx0 - margin));
const srcTop = Math.max(boxTop, Math.floor(my0 - margin));
const srcRight = Math.min(boxRight, Math.ceil(mx1 + margin));
const srcBottom = Math.min(boxBottom, Math.ceil(my1 + margin));
const srcW = srcRight - srcLeft;
const srcH = srcBottom - srcTop;
if (srcW >= 8 && srcH >= 8) {
// The model's grid comes from the crop's share of the FRAME and not
// from the pixels the frame is rendered at, so the same repair is the
// same repair at every zoom and at the export's own size — see
// INPAINT_REF_LONG. It is also finer than it used to be wherever the
// stage is small: a crop a tenth of a 768px preview is 77 pixels, and
// 77 pixels of context is what the model used to be asked to invent
// from.
const { w: mw, h: mh } = inpaintCropSize(
(srcW / width) * INPAINT_REF_LONG,
(srcH / height) * INPAINT_REF_LONG
);
const cropSurface = own(createSurface(mw, mh));
if (cropSurface) {
// Neural inpainting (inpaint-web): read the clean, unblurred original pixels
// from snap so the model is fed crisp scene context without Gaussian pre-blurring.
cropSurface.getCanvas().drawImageRect(
snap,
Skia.XYWHRect(srcLeft, srcTop, srcW, srcH),
Skia.XYWHRect(0, 0, mw, mh),
own(Skia.Paint())
);
flush(cropSurface);
const cropPixels = cropSurface.getCanvas().readPixels(0, 0, {
width: mw,
height: mh,
colorType: Skia.ColorType.RGBA_8888,
alphaType: Skia.AlphaType.Unpremul,
colorSpace: Skia.ColorSpace.SRGB,
});
if (cropPixels) {
const kx = mw / srcW;
const ky = mh / srcH;
const hole = new Uint8Array(mw * mh);
for (const s of cluster) {
const cx = (s.x * width - srcLeft) * kx;
const cy = (s.y * height - srcTop) * ky;
const rad = s.r * width * kx;
const x0 = Math.max(0, Math.floor(cx - rad));
const x1 = Math.min(mw - 1, Math.ceil(cx + rad));
const y0 = Math.max(0, Math.floor(cy - rad));
const y1 = Math.min(mh - 1, Math.ceil(cy + rad));
for (let y = y0; y <= y1; y++) {
const dy = y - cy;
const row = y * mw;
for (let x = x0; x <= x1; x++) {
const dx = x - cx;
if (dx * dx + dy * dy <= rad * rad) hole[row + x] = 1;
}
}
}
// Compute normalized patchKey so preview zoom in/out hits cache with 0ms overhead
const patchKey = `${mw}x${mh}_` + cluster.map((s) => `${s.x.toFixed(4)},${s.y.toFixed(4)},${s.r.toFixed(4)}`).join(';');
(hole as { patchKey?: string }).patchKey = patchKey;
const patch = await inpaintCrop(cropPixels, mw, mh, hole).catch(() => null);
if (patch) {
const patchSurface = own(createSurface(mw, mh));
if (patchSurface && patchSurface.getCanvas().writePixels(patch, mw, mh, 0, 0)) {
flush(patchSurface);
const patchImage = own(patchSurface.makeImageSnapshot());
if (patchImage) {
// The model's pixels under the spot, and nowhere else at
// all. What shape they are laid through is the join between
// the model's invention and the photo's real texture, and it
// is not the mask's shape: the model is the repair wherever
// it has one, so it owns the whole spot and gives way only in
// the sliver the mask itself fades over (HEAL_PATCH_STOPS,
// and the numbers that put it there are in heal.ts).
const patchSurface = own(createSurface(srcW, srcH));
if (patchSurface) {
const patchCanvas = patchSurface.getCanvas();
patchCanvas.clear(Skia.Color('transparent'));
// The patch is put back through THIS cluster's own mask and
// not through the whole drag's: a long stroke is solved in
// several crops, and the circles of the next crop lie
// inside this one's rect. Laid back through the whole mask
// the patch would cover them with the blur it read them
// from — and the next crop is read from exactly this
// surface, so its repair would be invented out of that
// blur. Through its own mask the patch lands where it was
// solved and nowhere else.
for (const s of cluster) {
const cx = s.x * width - srcLeft;
const cy = s.y * height - srcTop;
const rad = s.r * width;
const shader = own(
Skia.Shader.MakeRadialGradient(
[cx, cy],
rad,
new Float32Array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0]),
HEAL_PATCH_STOPS,
Skia.TileMode.Clamp
)
);
const spotPaint = own(Skia.Paint());
spotPaint.setAntiAlias(true);
spotPaint.setShader(shader);
patchCanvas.drawCircle(cx, cy, rad, spotPaint);
}
const patchPaint = own(Skia.Paint());
patchPaint.setAntiAlias(true);
patchPaint.setBlendMode(Skia.BlendMode.SrcIn);
patchCanvas.drawImageRect(
patchImage,
Skia.XYWHRect(0, 0, mw, mh),
Skia.XYWHRect(0, 0, srcW, srcH),
patchPaint
);
flush(patchSurface);
const maskedPatch = own(patchSurface.makeImageSnapshot());
if (maskedPatch) {
fillCanvas.drawImageRect(
maskedPatch,
Skia.XYWHRect(0, 0, srcW, srcH),
Skia.XYWHRect(srcLeft - boxLeft, srcTop - boxTop, srcW, srcH),
own(Skia.Paint())
);
}
}
}
}
}
}
}
}
}
flush(fillSurface);
const fillImage = own(fillSurface.makeImageSnapshot());
// And the fill back under the brush, in one draw: the frame, the fill and
// the mask as its three children, with the mask the weight the frame and
// the fill are mixed at — so the rim is the only place the two meet, and
// there they already agree.
if (fillImage && snap && maskImage && fillEffect) {
const shader = own(
fillEffect.makeShaderWithChildren([], [
own(imageShaderChild(snap)),
own(imageShaderChildAt(fillImage, boxLeft, boxTop)),
own(imageShaderChild(maskImage)),
])
);
if (shader) {
canvas.clear(Skia.Color('transparent'));
const healPaint = own(Skia.Paint());
healPaint.setShader(shader);
canvas.drawRect(Skia.XYWHRect(0, 0, width, height), healPaint);
}
}
}
// 6f. MOSAIC — the FX tab's hiding brush (see mosaic.ts). It comes after
// HEAL because hiding is the last thing the photo's own pixels are asked
// for: a repair under a hidden patch is work nobody will ever see, and the