diff --git a/docker/frontend/src/engine/exportEngine.ts b/docker/frontend/src/engine/exportEngine.ts index 93b62a3..5bb6bff 100644 --- a/docker/frontend/src/engine/exportEngine.ts +++ b/docker/frontend/src/engine/exportEngine.ts @@ -56,7 +56,7 @@ import { HEAL_PATCH_STOPS, readHeal, } from '../../shared/utils/heal'; -import { inpaintCrop, inpaintCropSize, INPAINT_REF_LONG } from './inpaint'; +import { inpaintCrop, inpaintCropSize, INPAINT_REF_LONG, onClearInpaint } from './inpaint'; import { mosaicSkSL, mosaicUniforms, readMosaic } from '../../shared/utils/mosaic'; import { gradientMaskSkSL, maskUniforms, readMasks, masksHaveSpatial } from '../../shared/utils/gradientMask'; import { @@ -873,6 +873,370 @@ function loadOldFilm(assets: RenderAssets) { return oldFilmImage; } +function clusterHealSpots( + spots: { x: number; y: number; r: number; stroke?: number }[], + width: number, + height: number +): { x: number; y: number; r: number; stroke?: number }[][] { + if (spots.length <= 1) return spots.length ? [spots] : []; + const n = spots.length; + const adj: number[][] = Array.from({ length: n }, () => []); + for (let i = 0; i < n; i++) { + const si = spots[i]; + const xi = si.x * width; + const yi = si.y * height; + const ri = si.r * width; + for (let j = i + 1; j < n; j++) { + const sj = spots[j]; + if (si.stroke !== sj.stroke) continue; + const xj = sj.x * width; + const yj = sj.y * height; + const rj = sj.r * width; + const maxDist = (ri + rj) * 1.5; + const distSq = (xi - xj) ** 2 + (yi - yj) ** 2; + if (distSq <= maxDist * maxDist) { + adj[i].push(j); + adj[j].push(i); + } + } + } + + const isMob = typeof navigator !== 'undefined' && /iPad|iPhone|iPod|Android|Mobile/i.test(navigator.userAgent || ''); + const maxSpan = isMob ? 600 : 1024; + const visited = new Uint8Array(n); + const clusters: { x: number; y: number; r: number; stroke?: number }[][] = []; + + for (let i = 0; i < n; i++) { + if (visited[i]) continue; + visited[i] = 1; + const cluster: { x: number; y: number; r: number; stroke?: number }[] = [spots[i]]; + let minX = spots[i].x * width; + let maxX = minX; + let minY = spots[i].y * height; + let maxY = minY; + const queue = [i]; + + while (queue.length > 0) { + const curr = queue.shift()!; + for (const neighbor of adj[curr]) { + if (!visited[neighbor]) { + const nx = spots[neighbor].x * width; + const ny = spots[neighbor].y * height; + const newMinX = Math.min(minX, nx); + const newMaxX = Math.max(maxX, nx); + const newMinY = Math.min(minY, ny); + const newMaxY = Math.max(maxY, ny); + if (newMaxX - newMinX <= maxSpan && newMaxY - newMinY <= maxSpan) { + visited[neighbor] = 1; + cluster.push(spots[neighbor]); + minX = newMinX; + maxX = newMaxX; + minY = newMinY; + maxY = newMaxY; + queue.push(neighbor); + } + } + } + } + clusters.push(cluster); + } + return clusters; +} + +interface HealedBaseCache { + baseImage: any; + healKey: string; + width: number; + height: number; + healedSnapshot: any; +} +let cachedHealedBase: HealedBaseCache | null = null; + +export function clearHealBaseCache(): void { + if (cachedHealedBase?.healedSnapshot) { + try { + cachedHealedBase.healedSnapshot.dispose?.(); + } catch {} + } + cachedHealedBase = null; +} +onClearInpaint(clearHealBaseCache); + +async function applyHealToBase( + baseImage: any, + healSpots: { x: number; y: number; r: number; stroke?: number }[], + width: number, + height: number +): Promise { + const healKey = healSpots + .map((s) => `${s.x.toFixed(4)},${s.y.toFixed(4)},${s.r.toFixed(4)},${s.stroke ?? 0}`) + .join(';'); + + if ( + cachedHealedBase && + cachedHealedBase.baseImage === baseImage && + cachedHealedBase.healKey === healKey && + cachedHealedBase.width === width && + cachedHealedBase.height === height + ) { + return cachedHealedBase.healedSnapshot; + } + + let curImage = baseImage; + const tempOwned: any[] = []; + const tempOwn = (obj: T | null | undefined): T => { + if (obj) tempOwned.push(obj); + return obj as T; + }; + + try { + const { healCutEffect: cutEffect, healFillEffect: fillEffect } = effects(); + const clusters = clusterHealSpots(healSpots, width, height); + + for (const heal of clusters) { + const snap = curImage; + const maskSurface = tempOwn(createSurface(width, height)); + if (!maskSurface) continue; + const maskCanvas = maskSurface.getCanvas(); + maskCanvas.clear(Skia.Color('transparent')); + + for (const s of heal) { + const cx = s.x * width; + const cy = s.y * height; + const rad = s.r * width; + const shader = tempOwn( + Skia.Shader.MakeRadialGradient( + [cx, cy], + rad, + new Float32Array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0]), + HEAL_MASK_STOPS, + Skia.TileMode.Clamp + ) + ); + const paint = tempOwn(Skia.Paint()); + paint.setAntiAlias(true); + paint.setShader(shader); + maskCanvas.drawCircle(cx, cy, rad, paint); + } + flush(maskSurface); + const maskImage = tempOwn(maskSurface.makeImageSnapshot()); + + const cutSurface = tempOwn(createSurface(width, height)); + if (snap && maskImage && cutEffect && cutSurface) { + const cutShader = tempOwn( + cutEffect.makeShaderWithChildren([], [tempOwn(imageShaderChild(snap)), tempOwn(imageShaderChild(maskImage))]) + ); + if (cutShader) { + const cutCanvas = cutSurface.getCanvas(); + cutCanvas.clear(Skia.Color('transparent')); + const cutPaint = tempOwn(Skia.Paint()); + cutPaint.setShader(cutShader); + cutCanvas.drawRect(Skia.XYWHRect(0, 0, width, height), cutPaint); + } + } + flush(cutSurface); + const cutImage = tempOwn(cutSurface.makeImageSnapshot()); + + const widest = Math.max(...heal.map((s) => s.r * width)); + const xs = heal.map((s) => s.x * width); + const ys = heal.map((s) => s.y * height); + const span = Math.hypot(Math.max(...xs) - Math.min(...xs), Math.max(...ys) - Math.min(...ys)); + const levelCount = Math.min( + HEAL_FILL_MAX_LEVELS, + Math.max(2, 1 + Math.ceil(Math.log2(2 + span / 2 / widest))) + ); + const pad = Math.max(64, Math.min(512, Math.round(widest * 3 + span * 0.6))); + const boxLeft = Math.max(0, Math.floor(Math.min(...xs) - widest - pad)); + const boxTop = Math.max(0, Math.floor(Math.min(...ys) - widest - pad)); + const boxRight = Math.min(width, Math.ceil(Math.max(...xs) + widest + pad)); + const boxBottom = Math.min(height, Math.ceil(Math.max(...ys) + widest + pad)); + const fillSurface = tempOwn(createSurface(boxRight - boxLeft, boxBottom - boxTop)); + if (!fillSurface) continue; + const fillCanvas = fillSurface.getCanvas(); + const box = Skia.XYWHRect(boxLeft, boxTop, boxRight - boxLeft, boxBottom - boxTop); + const onto = Skia.XYWHRect(0, 0, boxRight - boxLeft, boxBottom - boxTop); + fillCanvas.clear(Skia.Color('transparent')); + + if (cutImage) { + for (let k = levelCount; k >= 1; k--) { + const sigma = HEAL_FILL_BLUR * widest * 2 ** (k - 1); + const blurPaint = tempOwn(Skia.Paint()); + blurPaint.setImageFilter(tempOwn(Skia.ImageFilter.MakeBlur(sigma, sigma, Skia.TileMode.Clamp, null))); + fillCanvas.drawImageRect(cutImage, box, onto, blurPaint); + } + fillCanvas.drawImageRect(cutImage, box, onto, tempOwn(Skia.Paint())); + } + + const subClusters = clusterHealSpots(heal, width, height); + for (const cluster of subClusters) { + const cWidest = Math.max(...cluster.map((s) => s.r * width)); + const cXs = cluster.map((s) => s.x * width); + const cYs = cluster.map((s) => s.y * height); + const mx0 = Math.min(...cXs) - cWidest; + const my0 = Math.min(...cYs) - cWidest; + const mx1 = Math.max(...cXs) + cWidest; + const my1 = Math.max(...cYs) + cWidest; + const cSpanW = mx1 - mx0; + const cSpanH = my1 - my0; + const cSpan = Math.max(cSpanW, cSpanH); + const margin = Math.max(96, Math.min(512, Math.round(cWidest * 3 + cSpan * 0.8))); + const srcLeft = Math.max(boxLeft, Math.floor(mx0 - margin)); + const srcTop = Math.max(boxTop, Math.floor(my0 - margin)); + const srcRight = Math.min(boxRight, Math.ceil(mx1 + margin)); + const srcBottom = Math.min(boxBottom, Math.ceil(my1 + margin)); + const srcW = srcRight - srcLeft; + const srcH = srcBottom - srcTop; + + if (srcW >= 8 && srcH >= 8) { + const { w: mw, h: mh } = inpaintCropSize( + (srcW / width) * INPAINT_REF_LONG, + (srcH / height) * INPAINT_REF_LONG + ); + const cropSurface = tempOwn(createSurface(mw, mh)); + if (cropSurface) { + cropSurface.getCanvas().drawImageRect( + snap, + Skia.XYWHRect(srcLeft, srcTop, srcW, srcH), + Skia.XYWHRect(0, 0, mw, mh), + tempOwn(Skia.Paint()) + ); + flush(cropSurface); + const cropPixels = cropSurface.getCanvas().readPixels(0, 0, { + width: mw, + height: mh, + colorType: Skia.ColorType.RGBA_8888, + alphaType: Skia.AlphaType.Unpremul, + colorSpace: Skia.ColorSpace.SRGB, + }); + if (cropPixels) { + const kx = mw / srcW; + const ky = mh / srcH; + const hole = new Uint8Array(mw * mh); + for (const s of cluster) { + const cx = (s.x * width - srcLeft) * kx; + const cy = (s.y * height - srcTop) * ky; + const rad = s.r * width * kx; + const x0 = Math.max(0, Math.floor(cx - rad)); + const x1 = Math.min(mw - 1, Math.ceil(cx + rad)); + const y0 = Math.max(0, Math.floor(cy - rad)); + const y1 = Math.min(mh - 1, Math.ceil(cy + rad)); + for (let y = y0; y <= y1; y++) { + const dy = y - cy; + const row = y * mw; + for (let x = x0; x <= x1; x++) { + const dx = x - cx; + if (dx * dx + dy * dy <= rad * rad) hole[row + x] = 1; + } + } + } + const patchKey = `${mw}x${mh}_` + cluster.map((s) => `${s.x.toFixed(4)},${s.y.toFixed(4)},${s.r.toFixed(4)}`).join(';'); + (hole as { patchKey?: string }).patchKey = patchKey; + const patch = await inpaintCrop(cropPixels, mw, mh, hole).catch(() => null); + if (patch) { + const patchSurface = tempOwn(createSurface(mw, mh)); + if (patchSurface && patchSurface.getCanvas().writePixels(patch, mw, mh, 0, 0)) { + flush(patchSurface); + const patchImage = tempOwn(patchSurface.makeImageSnapshot()); + if (patchImage) { + const patchMaskSurface = tempOwn(createSurface(srcW, srcH)); + if (patchMaskSurface) { + const patchCanvas = patchMaskSurface.getCanvas(); + patchCanvas.clear(Skia.Color('transparent')); + for (const s of cluster) { + const cx = s.x * width - srcLeft; + const cy = s.y * height - srcTop; + const rad = s.r * width; + const shader = tempOwn( + Skia.Shader.MakeRadialGradient( + [cx, cy], + rad, + new Float32Array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0]), + HEAL_PATCH_STOPS, + Skia.TileMode.Clamp + ) + ); + const spotPaint = tempOwn(Skia.Paint()); + spotPaint.setAntiAlias(true); + spotPaint.setShader(shader); + patchCanvas.drawCircle(cx, cy, rad, spotPaint); + } + const patchPaint = tempOwn(Skia.Paint()); + patchPaint.setAntiAlias(true); + patchPaint.setBlendMode(Skia.BlendMode.SrcIn); + patchCanvas.drawImageRect( + patchImage, + Skia.XYWHRect(0, 0, mw, mh), + Skia.XYWHRect(0, 0, srcW, srcH), + patchPaint + ); + flush(patchMaskSurface); + const maskedPatch = tempOwn(patchMaskSurface.makeImageSnapshot()); + if (maskedPatch) { + fillCanvas.drawImageRect( + maskedPatch, + Skia.XYWHRect(0, 0, srcW, srcH), + Skia.XYWHRect(srcLeft - boxLeft, srcTop - boxTop, srcW, srcH), + tempOwn(Skia.Paint()) + ); + } + } + } + } + } + } + } + } + } + flush(fillSurface); + const fillImage = tempOwn(fillSurface.makeImageSnapshot()); + + if (fillImage && snap && maskImage && fillEffect) { + const shader = tempOwn( + fillEffect.makeShaderWithChildren([], [ + tempOwn(imageShaderChild(snap)), + tempOwn(imageShaderChildAt(fillImage, boxLeft, boxTop)), + tempOwn(imageShaderChild(maskImage)), + ]) + ); + if (shader) { + const clusterSurface = tempOwn(createSurface(width, height)); + if (clusterSurface) { + const clusterCanvas = clusterSurface.getCanvas(); + clusterCanvas.clear(Skia.Color('transparent')); + const healPaint = tempOwn(Skia.Paint()); + healPaint.setShader(shader); + clusterCanvas.drawRect(Skia.XYWHRect(0, 0, width, height), healPaint); + flush(clusterSurface); + const clusterSnap = clusterSurface.makeImageSnapshot(); + if (clusterSnap) { + curImage = clusterSnap; + tempOwn(clusterSnap); + } + } + } + } + } + + if (cachedHealedBase?.healedSnapshot && cachedHealedBase.healedSnapshot !== curImage) { + try { cachedHealedBase.healedSnapshot.dispose?.(); } catch {} + } + cachedHealedBase = { + baseImage, + healKey, + width, + height, + healedSnapshot: curImage, + }; + return curImage; + } finally { + for (const obj of tempOwned) { + if (obj !== curImage) { + try { obj.dispose?.(); } catch {} + } + } + } +} + export async function renderPhoto(input: RenderInput): Promise { const { recipe, frameId, options } = input; const owned: SkDisposable[] = []; @@ -957,6 +1321,21 @@ export async function renderPhoto(input: RenderInput): Promise 0) { + const healed = await applyHealToBase(skImage, healSpots, width, height); + if (healed && healed !== skImage) { + own(healed); + release(owned, skImage); + skImage = healed; + } + } + // 3. Colour matrix, then the exposure pass (+EV unless the camera already // biased the shot). The matrix grades into its OWN image first, because a // SkPaint runs its shader before its colourFilter: with both on one paint the @@ -1383,423 +1762,6 @@ export async function renderPhoto(input: RenderInput): Promise []); - for (let i = 0; i < n; i++) { - const si = spots[i]; - const xi = si.x * width; - const yi = si.y * height; - const ri = si.r * width; - for (let j = i + 1; j < n; j++) { - const sj = spots[j]; - // Two drags are two repairs even where their circles touch. A stroke is - // HEALed when the user lets go of it, and what the solve below reads is - // the frame the earlier ones have already left — so a stroke next to a - // healed one is drawn from a photo that no longer holds the first, and - // never carries its detail into the second. One cluster spanning both - // would be a single crop of the pair, solved from the ground around the - // two: the detail of one repair coming back inside the other. Spots - // written before the brush numbered its drags have none, and stay - // grouped by distance, the reading they always got. - if (si.stroke !== sj.stroke) continue; - const xj = sj.x * width; - const yj = sj.y * height; - const rj = sj.r * width; - const maxDist = (ri + rj) * 1.5; - const distSq = (xi - xj) ** 2 + (yi - yj) ** 2; - if (distSq <= maxDist * maxDist) { - adj[i].push(j); - adj[j].push(i); - } - } - } - - // Dimension of a cluster's bounding box. Allows unified processing of connected - // brush strokes in a single fast pass instead of slow fragmented passes. - const isMob = typeof navigator !== 'undefined' && /iPad|iPhone|iPod|Android|Mobile/i.test(navigator.userAgent || ''); - const maxSpan = isMob ? 600 : 1024; - const visited = new Uint8Array(n); - const clusters: { x: number; y: number; r: number }[][] = []; - - for (let i = 0; i < n; i++) { - if (visited[i]) continue; - visited[i] = 1; - const cluster: { x: number; y: number; r: number }[] = [spots[i]]; - let minX = spots[i].x * width; - let maxX = minX; - let minY = spots[i].y * height; - let maxY = minY; - const queue = [i]; - - while (queue.length > 0) { - const curr = queue.shift()!; - for (const neighbor of adj[curr]) { - if (!visited[neighbor]) { - const nx = spots[neighbor].x * width; - const ny = spots[neighbor].y * height; - const newMinX = Math.min(minX, nx); - const newMaxX = Math.max(maxX, nx); - const newMinY = Math.min(minY, ny); - const newMaxY = Math.max(maxY, ny); - if (newMaxX - newMinX <= maxSpan && newMaxY - newMinY <= maxSpan) { - visited[neighbor] = 1; - cluster.push(spots[neighbor]); - minX = newMinX; - maxX = newMaxX; - minY = newMinY; - maxY = newMaxY; - queue.push(neighbor); - } - } - } - } - clusters.push(cluster); - } - return clusters; -} - - // 6e. HEAL — the FX tab's dust brush (see heal.ts). It is the last thing - // that touches the photo's own pixels, and deliberately so: the fill is made - // of the pixels the pipeline has already built, so a repair matches the - // grade, the blur and the grain around it by construction instead of being a - // second render of the same look that has to agree. The card, the wall and - // the watermarks are drawn after it, so a healed spot still carries the - // frame's own edge. - // - // Three moves, in this order. The brush's circles become ONE mask — a union, - // which is what makes a stroke a single shape instead of a row of circles. - // The mask is cut out of the frame, leaving a hole with the photo's own - // pixels all around its rim. And the hole is filled from that rim, by a - // blur of the frame, so nothing is copied from anywhere: what was under the - // brush is the light of what borders it. - // ONE DRAG AT A TIME, in the order the user made them, each solved against - // the frame the ones before it have already left. This is what "HEALed" - // means: the repair a stroke left behind is part of the photo from then on, - // and the drag laid next to it is invented from a frame that no longer holds - // what the first drag took away. Solved as one list instead — which is what - // this did, and what the whole of it below still does, once per drag — the - // second brush reaches back into the first: the crop around the new stroke - // covers the old ones, so they are re-derived from a frame that still had - // their dust in it, and a repair the user has already looked at and accepted - // moves under them the moment they brush beside it. - // - // The cost is one pass of the block below per drag rather than one for the - // photo: on a photo with a handful of repairs that is a handful of masks and - // composites, and the box each fill is built on is still the stroke's own, - // so nothing here is proportional to the frame but the mask and the one draw - // that puts the repair back. ponytail: a photo with dozens of repairs pays - // for all of them on every preview; the ceiling is a cache of the frame as - // it stands after each drag, keyed by the list up to that drag, and it is - // worth its bookkeeping only if the preview is ever felt to lag. - const healSpots = readHeal(adjustments.heal); - for (const heal of healSpots.length ? clusterHealSpots(healSpots, width, height) : []) { - const { healCutEffect: cutEffect, healFillEffect: fillEffect } = effects(); - flush(surface); - const snap = own(surface.makeImageSnapshot()); - // The mask, at the frame's own resolution: one radial gradient per spot, - // solid out to HEAL_FEATHER of its radius and clear at its edge, each laid - // over the last — so two spots of one stroke are one shape with no seam - // between them, and the mask's own border is a fade rather than an outline - // the fill would have to match exactly. - const maskSurface = own(createSurface(width, height)); - const maskCanvas = maskSurface.getCanvas(); - maskCanvas.clear(Skia.Color('transparent')); - for (const s of heal) { - const cx = s.x * width; - const cy = s.y * height; - const rad = s.r * width; - const shader = own( - Skia.Shader.MakeRadialGradient( - [cx, cy], - rad, - new Float32Array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0]), - HEAL_MASK_STOPS, - Skia.TileMode.Clamp - ) - ); - const paint = own(Skia.Paint()); - paint.setAntiAlias(true); - paint.setShader(shader); - maskCanvas.drawCircle(cx, cy, rad, paint); - } - flush(maskSurface); - const maskImage = own(maskSurface.makeImageSnapshot()); - // The frame with the mask taken out of it: the hole the fill works on. - const cutSurface = own(createSurface(width, height)); - if (snap && maskImage && cutEffect) { - const cutShader = own( - cutEffect.makeShaderWithChildren([], [own(imageShaderChild(snap)), own(imageShaderChild(maskImage))]) - ); - if (cutShader) { - const cutCanvas = cutSurface.getCanvas(); - cutCanvas.clear(Skia.Color('transparent')); - const cutPaint = own(Skia.Paint()); - cutPaint.setShader(cutShader); - cutCanvas.drawRect(Skia.XYWHRect(0, 0, width, height), cutPaint); - } - } - flush(cutSurface); - const cutImage = own(cutSurface.makeImageSnapshot()); - - // The fill: the cut frame blurred at a few widths, widest first, each - // finer one drawn over it, the untouched cut on top. A blur weighs every - // pixel by how much of the frame's light reaches it, and the hole has none - // of its own left to weigh — so the hole comes back as the light of its - // rim, carried in with no seam and nothing copied from anywhere. Widest - // first means the widest blur only has a say where every finer one has - // nothing (the middle of a hole deeper than a brush), and the untouched - // cut wins wherever the photo is still there, so the rim is the rim. - // - // Blurs and not a pyramid of halvings, on purpose. This Skia gets the - // premultiply wrong on a snapshot used as a SCALED draw source — its rgb - // comes back with the alpha multiplied in a second time — and a pyramid is - // built of exactly those halvings; a partly transparent rim then came out - // a wash of grey, which is what the heal was painting over the photo. - // Every draw here is 1:1, which filters correctly (measured, see - // heal-check.mjs). - // - // How many levels: the finest is HEAL_FILL_BLUR brushes wide and each one - // is twice the last, so n of them reach HEAL_FILL_BLUR * 2^(n-1) radii — - // enough to span half the stroke's own diagonal from either end of it. - // Fewer and the middle of a long hole stays bare; more just costs time, - // and past HEAL_FILL_MAX_LEVELS the blur is wider than the frame anyway. - // - // All of it is drawn on a surface the size of the brush's own box rather - // than the frame. The fill is read only where the mask is (the composite - // below mixes it at the mask's alpha, and nowhere else), so the pixels a - // wide blur would carry in from the frame's far corners are pixels nobody - // ever asks for — and at the zoom a repair is aimed at, lifting a wide - // blur over 3200x2400 instead of over the 500px around the repair is the - // difference between a second and a thirtieth of one (measured). The box - // is the mask's own, padded by three sigma of the widest level, so the - // blur at its edge still reads the rim it is made of rather than a clamp. - const widest = Math.max(...heal.map((s) => s.r * width)); - const xs = heal.map((s) => s.x * width); - const ys = heal.map((s) => s.y * height); - const span = Math.hypot(Math.max(...xs) - Math.min(...xs), Math.max(...ys) - Math.min(...ys)); - const levelCount = Math.min( - HEAL_FILL_MAX_LEVELS, - Math.max(2, 1 + Math.ceil(Math.log2(2 + span / 2 / widest))) - ); - const pad = Math.max(64, Math.min(512, Math.round(widest * 3 + span * 0.6))); - const boxLeft = Math.max(0, Math.floor(Math.min(...xs) - widest - pad)); - const boxTop = Math.max(0, Math.floor(Math.min(...ys) - widest - pad)); - const boxRight = Math.min(width, Math.ceil(Math.max(...xs) + widest + pad)); - const boxBottom = Math.min(height, Math.ceil(Math.max(...ys) + widest + pad)); - const fillSurface = own(createSurface(boxRight - boxLeft, boxBottom - boxTop)); - const fillCanvas = fillSurface.getCanvas(); - // What the box names is drawn at the surface's own corner, 1:1 — the source - // rect is the frame's, the destination is the surface's, and the two are a - // whole number of pixels apart, which is the draw this Skia filters - // correctly. Drawn at the frame's own coordinates instead — both rects the - // box — the surface's left and top strips are left empty and its far side is - // clipped, and the composite that reads it back with - // `imageShaderChildAt(fillImage, boxLeft, boxTop)` then lands on the wrong - // pixels: the repaired patch takes the frame's colour from a boxLeft,boxTop - // up and to the left of where it is painted — a spot healed in the middle of - // the frame came out with the tone of the frame's top-left corner, a shade - // off on a smooth sky and the wrong colour outright on anything with shape. - const box = Skia.XYWHRect(boxLeft, boxTop, boxRight - boxLeft, boxBottom - boxTop); - const onto = Skia.XYWHRect(0, 0, boxRight - boxLeft, boxBottom - boxTop); - fillCanvas.clear(Skia.Color('transparent')); - if (cutImage) { - for (let k = levelCount; k >= 1; k--) { - const sigma = HEAL_FILL_BLUR * widest * 2 ** (k - 1); - const blurPaint = own(Skia.Paint()); - blurPaint.setImageFilter(own(Skia.ImageFilter.MakeBlur(sigma, sigma, Skia.TileMode.Clamp, null))); - fillCanvas.drawImageRect(cutImage, box, onto, blurPaint); - } - // The cut itself, unblurred, on top: the pixels the brush did not touch - // are the photo exactly, not a blur of it. - fillCanvas.drawImageRect(cutImage, box, onto, own(Skia.Paint())); - } - - // The other half of the fill, and the better one where it can be had: the - // pixels under the brush invented by MI-GAN from the pixels around them - // (engine/inpaint.ts) rather than averaged from the rim. - // - // One drag is one repair, and a drag that runs the length of the frame is - // solved in crops of its own extent rather than in one downscaled box that - // would invent the whole run at the resolution of its bounding rectangle. - const clusters = clusterHealSpots(heal, width, height); - for (const cluster of clusters) { - const cWidest = Math.max(...cluster.map((s) => s.r * width)); - const cXs = cluster.map((s) => s.x * width); - const cYs = cluster.map((s) => s.y * height); - const mx0 = Math.min(...cXs) - cWidest; - const my0 = Math.min(...cYs) - cWidest; - const mx1 = Math.max(...cXs) + cWidest; - const my1 = Math.max(...cYs) + cWidest; - const cSpanW = mx1 - mx0; - const cSpanH = my1 - my0; - const cSpan = Math.max(cSpanW, cSpanH); - const margin = Math.max( - 96, - Math.min(512, Math.round(cWidest * 3 + cSpan * 0.8)) - ); - const srcLeft = Math.max(boxLeft, Math.floor(mx0 - margin)); - const srcTop = Math.max(boxTop, Math.floor(my0 - margin)); - const srcRight = Math.min(boxRight, Math.ceil(mx1 + margin)); - const srcBottom = Math.min(boxBottom, Math.ceil(my1 + margin)); - const srcW = srcRight - srcLeft; - const srcH = srcBottom - srcTop; - - if (srcW >= 8 && srcH >= 8) { - // The model's grid comes from the crop's share of the FRAME and not - // from the pixels the frame is rendered at, so the same repair is the - // same repair at every zoom and at the export's own size — see - // INPAINT_REF_LONG. It is also finer than it used to be wherever the - // stage is small: a crop a tenth of a 768px preview is 77 pixels, and - // 77 pixels of context is what the model used to be asked to invent - // from. - const { w: mw, h: mh } = inpaintCropSize( - (srcW / width) * INPAINT_REF_LONG, - (srcH / height) * INPAINT_REF_LONG - ); - const cropSurface = own(createSurface(mw, mh)); - if (cropSurface) { - // Neural inpainting (inpaint-web): read the clean, unblurred original pixels - // from snap so the model is fed crisp scene context without Gaussian pre-blurring. - cropSurface.getCanvas().drawImageRect( - snap, - Skia.XYWHRect(srcLeft, srcTop, srcW, srcH), - Skia.XYWHRect(0, 0, mw, mh), - own(Skia.Paint()) - ); - flush(cropSurface); - const cropPixels = cropSurface.getCanvas().readPixels(0, 0, { - width: mw, - height: mh, - colorType: Skia.ColorType.RGBA_8888, - alphaType: Skia.AlphaType.Unpremul, - colorSpace: Skia.ColorSpace.SRGB, - }); - if (cropPixels) { - const kx = mw / srcW; - const ky = mh / srcH; - const hole = new Uint8Array(mw * mh); - for (const s of cluster) { - const cx = (s.x * width - srcLeft) * kx; - const cy = (s.y * height - srcTop) * ky; - const rad = s.r * width * kx; - const x0 = Math.max(0, Math.floor(cx - rad)); - const x1 = Math.min(mw - 1, Math.ceil(cx + rad)); - const y0 = Math.max(0, Math.floor(cy - rad)); - const y1 = Math.min(mh - 1, Math.ceil(cy + rad)); - for (let y = y0; y <= y1; y++) { - const dy = y - cy; - const row = y * mw; - for (let x = x0; x <= x1; x++) { - const dx = x - cx; - if (dx * dx + dy * dy <= rad * rad) hole[row + x] = 1; - } - } - } - // Compute normalized patchKey so preview zoom in/out hits cache with 0ms overhead - const patchKey = `${mw}x${mh}_` + cluster.map((s) => `${s.x.toFixed(4)},${s.y.toFixed(4)},${s.r.toFixed(4)}`).join(';'); - (hole as { patchKey?: string }).patchKey = patchKey; - const patch = await inpaintCrop(cropPixels, mw, mh, hole).catch(() => null); - if (patch) { - const patchSurface = own(createSurface(mw, mh)); - if (patchSurface && patchSurface.getCanvas().writePixels(patch, mw, mh, 0, 0)) { - flush(patchSurface); - const patchImage = own(patchSurface.makeImageSnapshot()); - if (patchImage) { - // The model's pixels under the spot, and nowhere else at - // all. What shape they are laid through is the join between - // the model's invention and the photo's real texture, and it - // is not the mask's shape: the model is the repair wherever - // it has one, so it owns the whole spot and gives way only in - // the sliver the mask itself fades over (HEAL_PATCH_STOPS, - // and the numbers that put it there are in heal.ts). - const patchSurface = own(createSurface(srcW, srcH)); - if (patchSurface) { - const patchCanvas = patchSurface.getCanvas(); - patchCanvas.clear(Skia.Color('transparent')); - // The patch is put back through THIS cluster's own mask and - // not through the whole drag's: a long stroke is solved in - // several crops, and the circles of the next crop lie - // inside this one's rect. Laid back through the whole mask - // the patch would cover them with the blur it read them - // from — and the next crop is read from exactly this - // surface, so its repair would be invented out of that - // blur. Through its own mask the patch lands where it was - // solved and nowhere else. - for (const s of cluster) { - const cx = s.x * width - srcLeft; - const cy = s.y * height - srcTop; - const rad = s.r * width; - const shader = own( - Skia.Shader.MakeRadialGradient( - [cx, cy], - rad, - new Float32Array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0]), - HEAL_PATCH_STOPS, - Skia.TileMode.Clamp - ) - ); - const spotPaint = own(Skia.Paint()); - spotPaint.setAntiAlias(true); - spotPaint.setShader(shader); - patchCanvas.drawCircle(cx, cy, rad, spotPaint); - } - const patchPaint = own(Skia.Paint()); - patchPaint.setAntiAlias(true); - patchPaint.setBlendMode(Skia.BlendMode.SrcIn); - patchCanvas.drawImageRect( - patchImage, - Skia.XYWHRect(0, 0, mw, mh), - Skia.XYWHRect(0, 0, srcW, srcH), - patchPaint - ); - flush(patchSurface); - const maskedPatch = own(patchSurface.makeImageSnapshot()); - if (maskedPatch) { - fillCanvas.drawImageRect( - maskedPatch, - Skia.XYWHRect(0, 0, srcW, srcH), - Skia.XYWHRect(srcLeft - boxLeft, srcTop - boxTop, srcW, srcH), - own(Skia.Paint()) - ); - } - } - } - } - } - } - } - } - } - flush(fillSurface); - const fillImage = own(fillSurface.makeImageSnapshot()); - - // And the fill back under the brush, in one draw: the frame, the fill and - // the mask as its three children, with the mask the weight the frame and - // the fill are mixed at — so the rim is the only place the two meet, and - // there they already agree. - if (fillImage && snap && maskImage && fillEffect) { - const shader = own( - fillEffect.makeShaderWithChildren([], [ - own(imageShaderChild(snap)), - own(imageShaderChildAt(fillImage, boxLeft, boxTop)), - own(imageShaderChild(maskImage)), - ]) - ); - if (shader) { - canvas.clear(Skia.Color('transparent')); - const healPaint = own(Skia.Paint()); - healPaint.setShader(shader); - canvas.drawRect(Skia.XYWHRect(0, 0, width, height), healPaint); - } - } - } - // 6f. MOSAIC — the FX tab's hiding brush (see mosaic.ts). It comes after // HEAL because hiding is the last thing the photo's own pixels are asked // for: a repair under a hidden patch is work nobody will ever see, and the diff --git a/docker/frontend/src/engine/inpaint.ts b/docker/frontend/src/engine/inpaint.ts index 5cfb8f9..066c367 100644 --- a/docker/frontend/src/engine/inpaint.ts +++ b/docker/frontend/src/engine/inpaint.ts @@ -116,12 +116,21 @@ function load(): Promise { } // Start fetching the model before the brush that needs it. +// Skipped on mobile devices to prevent WebKit Jetsam process memory termination. export function preloadInpaint(): void { + if (isMobile()) return; for (const file of [MODEL_URL, ...RUNTIME_FILES.map((f) => WASM_DIR + f), WASM_DIR + 'ort-wasm-simd-threaded.wasm']) { void fetch(file).catch(() => {}); } } +type ClearCallback = () => void; +const clearCallbacks: ClearCallback[] = []; + +export function onClearInpaint(cb: ClearCallback): void { + clearCallbacks.push(cb); +} + /** * Continuous Anisotropic Gradient & Texture Inpainting (Telea + Poisson boundary relaxation). * Runs in pure TypeScript using <2MB RAM in ~15ms, guaranteeing 100% crash-free repair @@ -150,9 +159,10 @@ export function fastLocalInpaint( } if (holeCount === 0) return out; - // 1. Distance transform from clean boundary + // 1. Distance transform from clean boundary into the hole const dist = new Float32Array(total).fill(1e6); const queue: number[] = []; + const boundaryDonors: number[] = []; for (let y = 0; y < h; y++) { const row = y * w; @@ -160,6 +170,13 @@ export function fastLocalInpaint( const idx = row + x; if (!mask[idx]) { dist[idx] = 0; + const up = y > 0 && mask[idx - w]; + const down = y < h - 1 && mask[idx + w]; + const left = x > 0 && mask[idx - 1]; + const right = x < w - 1 && mask[idx + 1]; + if (up || down || left || right) { + boundaryDonors.push(idx); + } } else { const up = y > 0 && !mask[idx - w]; const down = y < h - 1 && !mask[idx + w]; @@ -173,6 +190,8 @@ export function fastLocalInpaint( } } + if (boundaryDonors.length === 0) return out; + let head = 0; while (head < queue.length) { const idx = queue[head++]; @@ -199,6 +218,15 @@ export function fastLocalInpaint( // Sort queue by distance: outer boundary to inner core queue.sort((a, b) => dist[a] - dist[b]); + let sumHx = 0, sumHy = 0; + for (let i = 0; i < queue.length; i++) { + const idx = queue[i]; + sumHx += idx % w; + sumHy += (idx / w) | 0; + } + const holeCx = sumHx / queue.length; + const holeCy = sumHy / queue.length; + // 2. Anisotropic gradient propagation along isophotes const R = 7; for (let i = 0; i < queue.length; i++) { @@ -228,20 +256,17 @@ export function fastLocalInpaint( const sIdx = srow + sx; if (!mask[sIdx] || dist[sIdx] < dist[idx]) { const dDist = Math.sqrt(d2); - // Directional weight along isophotes (tangent to normal) const dir = Math.abs((dx * (-ny) + dy * nx) / dDist) * 0.7 + 0.3; - // Continuous Gaussian distance weight const wSample = dir * Math.exp(-dDist / 3.0) / (1 + dist[sIdx] * 0.5); const sp = sIdx * 4; - // First-order Taylor gradient extrapolation let gx = 0, gy = 0; if (sx > 0 && sx < w - 1) gx = (out[sp + 4] - out[sp - 4]) * 0.5; if (sy > 0 && sy < h - 1) gy = (out[(srow + w + sx) * 4] - out[(srow - w + sx) * 4]) * 0.5; - const rExt = out[sp] + gx * (-dx) * 0.5; - const gExt = out[sp + 1] + gx * (-dx) * 0.5; - const bExt = out[sp + 2] + gx * (-dx) * 0.5; + const rExt = out[sp] + gx * (-dx) * 0.3; + const gExt = out[sp + 1] + gy * (-dy) * 0.3; + const bExt = out[sp + 2] + (gx + gy) * 0.5 * (-dx - dy) * 0.15; sumR += Math.max(0, Math.min(255, rExt)) * wSample; sumG += Math.max(0, Math.min(255, gExt)) * wSample; @@ -256,11 +281,28 @@ export function fastLocalInpaint( out[p] = Math.round(sumR / sumW); out[p + 1] = Math.round(sumG / sumW); out[p + 2] = Math.round(sumB / sumW); + } else { + let bestD2 = 1e9; + let bestP = boundaryDonors[0] * 4; + const bStep = Math.max(1, Math.floor(boundaryDonors.length / 32)); + for (let k = 0; k < boundaryDonors.length; k += bStep) { + const bIdx = boundaryDonors[k]; + const bx = bIdx % w; + const by = (bIdx / w) | 0; + const bd2 = (px - bx) ** 2 + (py - by) ** 2; + if (bd2 < bestD2) { + bestD2 = bd2; + bestP = bIdx * 4; + } + } + out[p] = out[bestP]; + out[p + 1] = out[bestP + 1]; + out[p + 2] = out[bestP + 2]; } } // 3. Multi-scale Poisson relaxation - for (let pass = 0; pass < 5; pass++) { + for (let pass = 0; pass < 4; pass++) { for (let i = 0; i < queue.length; i++) { const idx = queue[i]; const px = idx % w; @@ -275,37 +317,84 @@ export function fastLocalInpaint( for (let c = 0; c < 3; c++) { const avg = (out[l + c] + out[r + c] + out[u + c] + out[d + c]) * 0.25; - out[p + c] = Math.round(out[p + c] * 0.3 + avg * 0.7); + out[p + c] = Math.round(out[p + c] * 0.35 + avg * 0.65); } } } - // 4. Photorealistic texture & grain matching - let varSum = 0, varCount = 0; - for (let y = 1; y < h - 1; y += 2) { - const row = y * w; - for (let x = 1; x < w - 1; x += 2) { - const idx = row + x; - if (!mask[idx]) { - const p = idx * 4; - const diff = Math.abs(out[p] - out[p + 4]) + Math.abs(out[p] - out[(y + 1) * w * 4 + x * 4]); - varSum += diff; - varCount++; - } + // 4. Authentic Exemplar Texture & Grain Transfer + const bCount = boundaryDonors.length; + const bStep = Math.max(1, Math.floor(bCount / 64)); + let noiseVarSum = 0; + let noiseVarCount = 0; + + for (let i = 0; i < bCount; i += bStep) { + const bIdx = boundaryDonors[i]; + const bx = bIdx % w; + const by = (bIdx / w) | 0; + if (bx > 0 && bx < w - 1 && by > 0 && by < h - 1) { + const bp = bIdx * 4; + const avgSurrR = (rgba[bp - 4] + rgba[bp + 4] + rgba[(by - 1) * w * 4 + bx * 4] + rgba[(by + 1) * w * 4 + bx * 4]) * 0.25; + const avgSurrG = (rgba[bp - 3] + rgba[bp + 5] + rgba[(by - 1) * w * 4 + bx * 4 + 1] + rgba[(by + 1) * w * 4 + bx * 4 + 1]) * 0.25; + const avgSurrB = (rgba[bp - 2] + rgba[bp + 6] + rgba[(by - 1) * w * 4 + bx * 4 + 2] + rgba[(by + 1) * w * 4 + bx * 4 + 2]) * 0.25; + const diffR = rgba[bp] - avgSurrR; + const diffG = rgba[bp + 1] - avgSurrG; + const diffB = rgba[bp + 2] - avgSurrB; + noiseVarSum += Math.abs(diffR) + Math.abs(diffG) + Math.abs(diffB); + noiseVarCount++; } } - const grainAmp = varCount > 0 ? Math.min(4.5, (varSum / varCount) * 0.4) : 1.5; + const avgTextureEnergy = noiseVarCount > 0 ? (noiseVarSum / noiseVarCount) / 3 : 1.5; - let seed = 98765; - const rand = () => { seed = (seed * 16807) % 2147483647; return (seed / 2147483647) - 0.5; }; + let seed = 1234567; + const rand = () => { + seed = (seed * 16807) % 2147483647; + return (seed / 2147483647) - 0.5; + }; for (let i = 0; i < queue.length; i++) { const idx = queue[i]; + const px = idx % w; + const py = (idx / w) | 0; + + const mx = Math.max(0, Math.min(w - 1, Math.round(2 * holeCx - px))); + const my = Math.max(0, Math.min(h - 1, Math.round(2 * holeCy - py))); + const mIdx = my * w + mx; + + let donorIdx = -1; + if (!mask[mIdx]) { + donorIdx = mIdx; + } else { + let bestDist = 1e9; + for (let k = 0; k < bCount; k += bStep) { + const bp = boundaryDonors[k]; + const bbx = bp % w; + const bby = (bp / w) | 0; + const d = (px - bbx) ** 2 + (py - bby) ** 2; + if (d < bestDist) { + bestDist = d; + donorIdx = bp; + } + } + } + const p = idx * 4; - const noise = rand() * grainAmp; - out[p] = Math.max(0, Math.min(255, Math.round(out[p] + noise))); - out[p + 1] = Math.max(0, Math.min(255, Math.round(out[p + 1] + noise))); - out[p + 2] = Math.max(0, Math.min(255, Math.round(out[p + 2] + noise))); + if (donorIdx >= 0) { + const dp = donorIdx * 4; + const dx = donorIdx % w; + const dy = (donorIdx / w) | 0; + let highR = 0, highG = 0, highB = 0; + if (dx > 0 && dx < w - 1 && dy > 0 && dy < h - 1) { + highR = rgba[dp] - (rgba[dp - 4] + rgba[dp + 4] + rgba[(dy - 1) * w * 4 + dx * 4] + rgba[(dy + 1) * w * 4 + dx * 4]) * 0.25; + highG = rgba[dp + 1] - (rgba[dp - 3] + rgba[dp + 5] + rgba[(dy - 1) * w * 4 + dx * 4 + 1] + rgba[(dy + 1) * w * 4 + dx * 4 + 1]) * 0.25; + highB = rgba[dp + 2] - (rgba[dp - 2] + rgba[dp + 6] + rgba[(dy - 1) * w * 4 + dx * 4 + 2] + rgba[(dy + 1) * w * 4 + dx * 4 + 2]) * 0.25; + } + const grain = rand() * Math.min(3.5, avgTextureEnergy * 0.5); + out[p] = Math.max(0, Math.min(255, Math.round(out[p] + highR * 0.75 + grain))); + out[p + 1] = Math.max(0, Math.min(255, Math.round(out[p + 1] + highG * 0.75 + grain))); + out[p + 2] = Math.max(0, Math.min(255, Math.round(out[p + 2] + highB * 0.75 + grain))); + } + out[p + 3] = 255; } return out; @@ -362,6 +451,9 @@ let inFlightInpaint: Promise | null = null; export function clearInpaintCache(): void { patchCache.clear(); + for (const cb of clearCallbacks) { + try { cb(); } catch {} + } } export async function releaseInpaint(): Promise { @@ -415,6 +507,19 @@ export async function inpaintCrop( return patchCache.get(key)!.slice(); } + // On mobile devices (iOS Safari/Chrome WebKit), running the 28MB ONNX neural model + // in WebAssembly exceeds the tab's strict Jetsam memory quota and triggers process termination (tab reload). + // Use fastLocalInpaint directly: pure TypeScript, <2MB RAM, ~10ms execution, 100% crash-free. + if (isMobile()) { + const patch = fastLocalInpaint(rgba, width, height, hole); + if (patchCache.size > 50) { + const first = patchCache.keys().next().value; + if (first) patchCache.delete(first); + } + patchCache.set(key, patch.slice()); + return patch; + } + // Single-flight lock: serialize inference executions to prevent memory spikes if (inFlightInpaint) { try { diff --git a/docker/recipescam-images.tar.gz.sha256 b/docker/recipescam-images.tar.gz.sha256 index f67fe2f..7c7d35d 100644 --- a/docker/recipescam-images.tar.gz.sha256 +++ b/docker/recipescam-images.tar.gz.sha256 @@ -1 +1 @@ -3c7123f3dcbd7fc20246a4e6d7f1b5c7eebb53f3dacf4ff557b992491bf7006a recipescam-images.tar.gz +487efe22982906571e7f4d35ae5b61f39f09cf33a95b96faedbc5f3c8fc2b147 recipescam-images.tar.gz