RecipesCam/docker/frontend/shared/utils/heal.ts

import type { HealSpot } from '../types';

// FX tab > HEAL — the dust brush, and the fill behind it.
//
// What the brush paints is a mask: the circles it laid down, all of them at
// once, as ONE shape. What fills that shape is not borrowed from anywhere — the
// renderer takes the photo around the mask's rim and works inward, so the
// repair is made of the light and the colour that already border it. That is
// the whole of the change from a patch brush: a stroke of twenty overlapping
// spots used to be twenty independent repairs, each pasting its own patch and
// matching its own rim, which is exactly what a run of circles looks like on the
// photo; one mask and one solve have no circles in them to see.
//
// Non-destructive like everything else here: the recipe keeps the circles, and
// the preview and the export run the same two passes over them.
//
// x/y are fractions of the render's width and height, r of its width. The
// fractions are what make one set of spots land on the preview and on the
// full-size export, and keep a spot round whatever the photo's shape.

// The brush's radius, as a fraction of the photo's width — where this tool
// starts, at the sensor-dust end, where a speck is a few thousandths of the
// frame. The bounds and the wheel that move it between them belong to the brush
// both FX tools share (brush.ts).
export const HEAL_DEFAULT_R = 0.012;

// How much of a spot is painted at full strength, as a fraction of its radius:
// out to 75% the mask is solid to completely eliminate defects, and the outer 25%
// is a smooth Hermite fade (smoothstep) to blend invisibly into the surrounding photo.
export const HEAL_FEATHER = 0.75;

// The finest of the fill's blurs, as a multiple of the widest brush in the
// stroke. The light of the rim has to reach the middle of the hole, so the
// reach is about a brush wide and no more: much wider and the fill carries the
// light of places the repair has no business reading (a red card three radii
// away bled into the hole at 1.2), much narrower and a wide hole is left bare.
// Measured in heal-check.mjs; 1.0 is the widest that passes.
export const HEAL_FILL_BLUR = 1.0;

// How many widths of blur the fill is built from, each twice the last. The
// finest is HEAL_FILL_BLUR brushes wide; the widest of n of them reaches
// HEAL_FILL_BLUR * 2^(n-1) brush radii. The count has to grow with the depth of
// the hole — a stroke two brushes wide comes out hollow with too few levels and
// a blurred red card with too many — so the renderer picks it per stroke and
// this is only the ceiling. Measured in heal-check.mjs and tmp-blob-probe.mjs:
// a hole two brushes across needs four levels, one ten across needs six.
export const HEAL_FILL_MAX_LEVELS = 8;

// The stored spots, made readable: numbers, inside the frame. Everything below
// reads a recipe through this, so a hand-written or older file cannot produce a
// spot the brush and the renderer disagree about. The patch the older brush
// stored (sx/sy) is not read: a repair is no longer a copy of another place.
export function readHeal(heal: HealSpot[] | undefined): HealSpot[] {
  if (!Array.isArray(heal)) return [];
  return heal
    .map((s) => {
      const spot: HealSpot = {
        x: Math.min(1, Math.max(0, num(s?.x, 0))),
        y: Math.min(1, Math.max(0, num(s?.y, 0))),
        r: Math.max(0, num(s?.r, 0)),
      };
      // Which drag the spot came from is carried through. The renderer solves
      // each drag as its own repair and in the order they were drawn, so each
      // one reads the repairs already made next to it rather than the photo
      // they were made over (exportEngine's heal step), and it can only do that
      // if the record of the gesture survives being read.
      if (typeof s?.stroke === 'number' && Number.isFinite(s.stroke)) spot.stroke = s.stroke;
      return spot;
    })
    .filter((s) => s.r > 0);
}

function num(v: unknown, fallback: number): number {
  const n = Number(v);
  return Number.isFinite(n) ? n : fallback;
}

// The first of the fill's two passes: the photo with the painted pixels taken
// out of it. The colour of every pixel is kept and its alpha drops to nothing
// over the brush, which is what turns the mask into a hole the second pass can
// see the rim of. Written as premultiplied — rgb times that alpha — because
// that is what every surface here holds, and the blur below is built on it: a
// blur of premultiplied pixels is a weighted average of the pixels that are
// still there, with the hole weighing nothing.
//
// The cut is the WHOLE stroke, and it is a step rather than the mask's own fade.
// The fill below reads the rim of this hole, so whatever is left standing inside
// it is weighed into the repair. Cut at the mask's fade instead — `a = 1 -
// mask.a`, the obvious reading of "take the mask out" — and the outer quarter of
// every spot stays behind at partial weight. A brush aimed at a blemish has the
// blemish's own rim in exactly that band, so the repair came back in the colour
// of the thing being repaired: on a grey ground with a red blemish, the obvious
// cut is 4.1 levels off the ground and the step is 1.1 (measured, the cut
// variants in heal-colour-pass.mjs). The threshold is lightcraft's own: the
// pixels a heal may read its tone from are the ones at alpha < 0.02 (spots.rs).
export const HEAL_CUT_SKSL = `
uniform shader img;
uniform shader mask;

half4 main(float2 pos) {
  half4 c = img.eval(pos);
  half a = mask.eval(pos).a < 0.02 ? 1.0 : 0.0;
  half3 straight = c.a > 0.0001 ? c.rgb / c.a : c.rgb;
  return half4(straight * a, a);
}
`;

// The second pass: the fill laid back under the brush.
//
// The weight the fill is mixed in at is the mask's own ramp, read over the WHOLE
// of it: m is 1 out to HEAL_FEATHER of the radius and runs to 0 at its edge, so
// the fill arrives exactly as far as the mask says it may. Reaching full weight
// at 35% of the ramp instead — smoothstep(0, 0.35, m), which is what this shader
// did first — crosses the fade in a third of the room the mask gives it, and the
// repair then meets the photo along a line. Measured on the probe
// (probe-seam.mjs, one stroke across a photo's own detail, ROW 0.75 and 0.85),
// counting the pixels outside the brush that moved by more than six levels: 338
// and 392 along that line, by up to 73 and 148 levels, 13 and 39 of them more
// than a pixel and a half outside the rim. Read over the whole ramp: 66 and 57,
// by up to 11 and 45, and none of them more than a pixel and a half outside.
export const HEAL_FILL_SKSL = `
uniform shader img;
uniform shader fill;
uniform shader mask;

half4 main(float2 pos) {
  half4 c = img.eval(pos);
  half m = mask.eval(pos).a;
  if (m <= 0.0) return c;
  half4 f = fill.eval(pos);
  half3 here = c.a > 0.0001 ? c.rgb / c.a : c.rgb;
  half3 made = f.a > 0.001 ? f.rgb / f.a : here;
  half sm = smoothstep(0.0, 1.0, m);
  return half4(mix(here, made, sm) * c.a, c.a);
}
`;

// How the mask is painted onto a surface, as the stops of one radial gradient
// per spot: solid white out to HEAL_FEATHER of the radius, nothing at its edge,
// a straight ramp between. The renderer builds the gradient per spot (Skia has
// no one paint for a list of circles) and draws each at its own radius, so a
// brush turned down to three pixels and one at a quarter of the frame feather by
// the same fraction of themselves.
export const HEAL_MASK_STOPS: number[] = [0, HEAL_FEATHER, 1];

// The same stops for the model's own pixels, and they are not the mask's. The
// model is the repair wherever it has one, so its shape is the whole spot and
// the last fraction of a radius is the only place it gives way. At
// HEAL_MASK_STOPS instead — the model solid out to 0.75 of the radius and faded
// out by the rim — the band between was the FILL's, and the fill is built from
// the frame's own light over the whole of its box (a stroke a third of the frame
// long is filled by blurs up to sixteen brush radii wide, so what it brings into
// that band is the box's average and not the rim's).
//
// Measured (probe-seam.mjs, one stroke of seven spots across the sky, `d` over
// the spots' own circles between the frame before the repair and after it): the
// band came back 8 to 11 levels below the photo it was made in — a ring, on the
// one line a repair is looked at along. With the model owning the band: 0.71
// levels of mean change over the whole repair, 8.5 at its worst pixel, and the
// edge strength at the rim after the repair 0.84 where the photo's own edges
// around it are 0.5 (1.52 before). Past 0.92 the numbers stop moving (0.73 /
// 8.7 / 0.88 at 0.995), so the model does own the band and what is left there is
// the shape of the spot, not the fill under it.
export const HEAL_PATCH_STOPS: number[] = [0, 0.92, 1];