RecipesCam/docker/frontend/scripts/heal-check.mjs

// HEAL is a mask and a fill now, not a patch pasted under each spot. Four
// things can go wrong and none of them throws: the circles of one stroke can
// stay separate shapes (which is what a row of round grey blobs on the photo
// is), the fill can fail to carry the light of the photo around it (a repair
// that reads as a mark of its own), the search can still borrow a patch from
// somewhere (the card three radii away, which is the brush the user complained
// about), or the composite can touch pixels it was never given (a smear across
// the whole frame). So the mask, the fill and the untouched ground are all
// measured here, on synthetic photos whose answer is known, with CanvasKit
// running the same two passes exportEngine runs.
//
// The fill now has a second, better author where one can be had — MI-GAN, over
// the mask (engine/inpaint.ts) — and this check deliberately measures the other
// one: the blur that stands in when the model cannot be had, which is the case
// on a machine without the runtime and the case this file can run offline. What
// is asserted about the model is only that the pass is still wired in, and with
// which polarity.
//
//   node scripts/heal-check.mjs
import assert from 'node:assert/strict';
import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import ts from 'typescript';
import CanvasKitInit from 'canvaskit-wasm/bin/full/canvaskit.js';

const transpile = (path) =>
  ts.transpileModule(readFileSync(new URL(path, import.meta.url), 'utf8'), {
    compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2022 },
  }).outputText;

const dir = mkdtempSync(join(tmpdir(), 'heal-check-'));
writeFileSync(join(dir, 'heal.mjs'), transpile('../shared/utils/heal.ts'));
const { readHeal, HEAL_MASK_STOPS, HEAL_FILL_BLUR, HEAL_FILL_MAX_LEVELS, HEAL_CUT_SKSL, HEAL_FILL_SKSL } = await import(
  pathToFileURL(join(dir, 'heal.mjs')).href
);

// The other half of the check: the passes below are these two shaders, so the
// engine has to be compiling them — and it must not still be building the
// per-spot shader this replaced.
const engine = readFileSync(new URL('../src/engine/exportEngine.ts', import.meta.url), 'utf8');
assert.match(engine, /healCutEffect = Skia\.RuntimeEffect\.Make\(HEAL_CUT_SKSL\)/, 'exportEngine does not compile HEAL_CUT_SKSL');
assert.match(engine, /healFillEffect = Skia\.RuntimeEffect\.Make\(HEAL_FILL_SKSL\)/, 'exportEngine does not compile HEAL_FILL_SKSL');
assert.doesNotMatch(engine, /healSkSL\(/, 'the per-spot heal shader is still in exportEngine');
assert.doesNotMatch(engine, /healUniforms\(/, 'the per-spot uniform block is still in exportEngine');
// ...and the fill's level count is the one this check builds, not a constant.
assert.match(engine, /HEAL_FILL_MAX_LEVELS/, 'exportEngine does not cap the fill levels at HEAL_FILL_MAX_LEVELS');
assert.match(engine, /1 \+ Math\.ceil\(Math\.log2\(2 \+ span \/ 2 \/ widest\)\)/, 'exportEngine no longer scales the fill levels with the depth of the hole');
// ...and the fill is built on the brush's box, read back at the frame's origin.
assert.match(engine, /imageShaderChildAt\(fillImage, boxLeft, boxTop\)/, 'the fill child is read at its own origin instead of the frame\'s');
// ...and the box the fill is built on is drawn at the surface's own corner, which
// is where that child reads it. Drawn at the frame's coordinates instead, the two
// disagree by the box's corner, and the repair comes back a patch of the frame
// from up and to the left of the spot (the case at the foot of this file).
assert.match(engine, /drawImageRect\(cutImage, box, onto, /, 'the fill is drawn at the box\'s frame coordinates, not at the surface\'s own corner');

// The model half is wired into the same pass: the masked crop is asked for, the
// answer is written into the fill surface over the box, and the mask it is
// handed is built from the brush's own circles. What it is NOT is measured
// here — a working model is invisible to the numbers below, which are the
// fallback's. The polarity is MI-GAN's, 255 KEEP and 0 HOLE, and it is the one
// thing that, got backwards, repaints the frame and leaves the hole (measured in
// engine/inpaint.ts; the probe is gone).
const inpaint = readFileSync(new URL('../src/engine/inpaint.ts', import.meta.url), 'utf8');
assert.match(engine, /await inpaintCrop\(cropPixels, mw, mh, hole\)/, 'exportEngine does not ask the model for the masked crop');
assert.match(engine, /patchSurface\.getCanvas\(\)\.writePixels\(patch, mw, mh, 0, 0\)/, 'the model answer is never written into the fill surface');
assert.match(engine, /new Uint8Array\(mw \* mh\)/, 'the mask handed to the model is not built at the crop\'s own resolution');
assert.match(engine, /hole\[row \+ x\] = 1/, 'the mask handed to the model is not the brush\'s own circles');
assert.match(inpaint, /fill\(255\)/, 'the model mask no longer starts at KEEP');
assert.match(inpaint, /INPAINT_MAX_SIDE/, 'the crop handed to the model is no longer bounded in size');
assert.match(inpaint, /inpaintCropSize/, 'nothing tells the renderer the resolution the model will run at');

// --- the stored spots --------------------------------------------------------
// A recipe written by the older brush carries sx/sy. It still loads: those two
// numbers are no longer read, and nothing else about the spot moves.
assert.deepEqual(readHeal([{ x: 0.5, y: 0.5, r: 0.02, sx: 0.9, sy: 0.1 }]), [{ x: 0.5, y: 0.5, r: 0.02 }]);
// Out of frame, non-numeric, or a radius of nothing: a spot the brush and the
// renderer would disagree about is dropped either way.
assert.deepEqual(readHeal([{ x: 2, y: -1, r: 0.05 }, { x: 'a', y: 0, r: 0.05 }, { x: 0.5, y: 0.5, r: 0 }]), [
  { x: 1, y: 0, r: 0.05 },
  { x: 0, y: 0, r: 0.05 },
]);

const CanvasKit = await CanvasKitInit({
  locateFile: () => fileURLToPath(new URL('../node_modules/canvaskit-wasm/bin/full/canvaskit.wasm', import.meta.url)),
});

// The frame every pass below is built on. One size for most of the check; the
// last case needs a bigger one, which is the whole point of it (see there), so
// these two are the frame and the functions here read them.
let W = 160;
let H = 160;
const GREY = 118;
const DUST = 38;
const R = 0.05; // the brush: an 8px radius on this frame
// A stroke of nine spots, one every 0.6 of a radius, running down the frame.
const spots = Array.from({ length: 9 }, (_, i) => ({ x: 0.5, y: (48 + i * R * 0.6 * W) / H, r: R }));
const cx = 0.5 * W;
const cy = 48 + 4 * R * 0.6 * W;
// Three radii to the right of the stroke: where the old search looked.
const CARD_X = Math.round(cx + 3.2 * R * W);

const shadeFlat = (x, y) => {
  if (Math.abs(x - CARD_X) < 6 && Math.abs(y - cy) < 6) return [220, 30, 30];
  if (Math.hypot(x - cx, y - cy) < 6) return [DUST, DUST, DUST];
  return [GREY, GREY, GREY];
};
const shadeRamp = (x) => {
  const v = 40 + (120 * x) / (W - 1);
  return [v, v, v];
};

function bytesOf(shade) {
  const rgba = new Uint8Array(W * H * 4);
  for (let y = 0; y < H; y++) {
    for (let x = 0; x < W; x++) {
      const [r, g, b] = shade(x, y);
      const i = (y * W + x) * 4;
      rgba[i] = r;
      rgba[i + 1] = g;
      rgba[i + 2] = b;
      rgba[i + 3] = 255;
    }
  }
  return rgba;
}
const imageOf = (rgba) =>
  CanvasKit.MakeImage(
    {
      width: W,
      height: H,
      colorType: CanvasKit.ColorType.RGBA_8888,
      alphaType: CanvasKit.AlphaType.Unpremul,
      colorSpace: CanvasKit.ColorSpace.SRGB,
    },
    rgba,
    W * 4
  );
const surface = (w, h) => {
  const s = CanvasKit.MakeSurface(w, h);
  s.getCanvas().clear(CanvasKit.TRANSPARENT);
  return s;
};
const pixelsOf = (image) => {
  const s = surface(image.width(), image.height());
  s.getCanvas().drawImage(image, 0, 0);
  return s.getCanvas().readPixels(0, 0, {
    width: image.width(),
    height: image.height(),
    colorType: CanvasKit.ColorType.RGBA_8888,
    alphaType: CanvasKit.AlphaType.Unpremul,
    colorSpace: CanvasKit.ColorSpace.SRGB,
  });
};
const asChild = (image) =>
  image.makeShaderOptions(CanvasKit.TileMode.Clamp, CanvasKit.TileMode.Clamp, CanvasKit.FilterMode.Linear, CanvasKit.MipmapMode.None);
// The fill's own child, read in the frame's coordinates: the fill is built on
// the brush's box (below), and a whole-pixel shift is what the engine does.
const asChildAt = (image, x, y) =>
  image.makeShaderOptions(
    CanvasKit.TileMode.Clamp,
    CanvasKit.TileMode.Clamp,
    CanvasKit.FilterMode.Linear,
    CanvasKit.MipmapMode.None,
    [1, 0, x, 0, 1, y, 0, 0, 1]
  );

// The brush's circles as ONE mask: a radial gradient per spot, solid out to
// HEAL_FEATHER of its radius and clear at its edge, each laid over the last.
function maskOf(spotsIn) {
  const surf = surface(W, H);
  const canvas = surf.getCanvas();
  for (const s of spotsIn) {
    const rad = s.r * W;
    const shader = CanvasKit.Shader.MakeRadialGradient(
      [s.x * W, s.y * H],
      rad,
      new Float32Array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0]),
      HEAL_MASK_STOPS,
      CanvasKit.TileMode.Clamp
    );
    const paint = new CanvasKit.Paint();
    paint.setAntiAlias(true);
    paint.setShader(shader);
    canvas.drawCircle(s.x * W, s.y * H, rad, paint);
  }
  return surf.makeImageSnapshot();
}

// exportEngine's 6e, pass for pass: cut, the blurred fill, and the fill laid
// back under the brush.
function healPass(rgba, spotsIn) {
  const src = imageOf(rgba);
  const mask = maskOf(spotsIn);
  const cutEffect = CanvasKit.RuntimeEffect.Make(HEAL_CUT_SKSL);
  assert.ok(cutEffect, 'HEAL_CUT_SKSL does not compile');
  const cut = surface(W, H);
  const cutPaint = new CanvasKit.Paint();
  cutPaint.setShader(cutEffect.makeShaderWithChildren([], [asChild(src), asChild(mask)]));
  cut.getCanvas().drawRect(CanvasKit.XYWHRect(0, 0, W, H), cutPaint);

  // The fill: the cut frame blurred at a few widths, widest first, each finer
  // one drawn over it, the untouched cut on top. A blur is a weighted mean of
  // the premultiplied pixels around each pixel, and the hole weighs nothing, so
  // the hole comes back as the light of its rim. Widest first means the widest
  // blur only has a say where every finer one has nothing — the middle of a hole
  // deeper than a brush — and the untouched cut wins wherever the photo is still
  // there, so the rim is the rim exactly.
  //
  // Blurs and not the pyramid of halvings the engine used to build: this Skia
  // gets the premultiply wrong on a snapshot drawn with a SCALE — its rgb comes
  // back with the alpha multiplied in a second time — and a pyramid is made of
  // exactly those halvings, so a partly transparent rim came back a wash of
  // grey. Every draw here is 1:1, which filters correctly.
  const cutImage = cut.makeImageSnapshot();
  // The count and the widths are exportEngine's own, to the letter: a check
  // that picks its own levels is checking a renderer nobody ships. So is the
  // surface the fill is built on — the brush's own box, padded by three sigma of
  // the widest level, rather than the frame.
  const widest = Math.max(...spotsIn.map((s) => s.r * W));
  const xs = spotsIn.map((s) => s.x * W);
  const ys = spotsIn.map((s) => s.y * H);
  const span = Math.hypot(Math.max(...xs) - Math.min(...xs), Math.max(...ys) - Math.min(...ys));
  const levelCount = Math.min(
    HEAL_FILL_MAX_LEVELS,
    Math.max(2, 1 + Math.ceil(Math.log2(2 + span / 2 / widest)))
  );
  const pad = 3 * HEAL_FILL_BLUR * widest * 2 ** (levelCount - 1);
  const boxLeft = Math.max(0, Math.floor(Math.min(...xs) - widest - pad));
  const boxTop = Math.max(0, Math.floor(Math.min(...ys) - widest - pad));
  const boxRight = Math.min(W, Math.ceil(Math.max(...xs) + widest + pad));
  const boxBottom = Math.min(H, Math.ceil(Math.max(...ys) + widest + pad));
  const box = CanvasKit.XYWHRect(boxLeft, boxTop, boxRight - boxLeft, boxBottom - boxTop);
  // ...and the box's region is drawn at the surface's OWN corner, which is what
  // the child below reads it back with. Both rects the `box` — the frame's own
  // coordinates on a surface the size of the box — is the draw that leaves the
  // surface's left and top strips empty, clips its far side, and hands the
  // composite a patch of the frame from boxLeft, boxTop up and to the left of the
  // spot: a repair the wrong shade, on a ramp a far-off one. `box, onto`, and the
  // case at the foot of this file is what keeps it that way.
  const onto = CanvasKit.XYWHRect(0, 0, boxRight - boxLeft, boxBottom - boxTop);
  const fillBox = surface(boxRight - boxLeft, boxBottom - boxTop);
  for (let k = levelCount; k >= 1; k--) {
    const sigma = HEAL_FILL_BLUR * widest * 2 ** (k - 1);
    const blurPaint = new CanvasKit.Paint();
    blurPaint.setImageFilter(CanvasKit.ImageFilter.MakeBlur(sigma, sigma, CanvasKit.TileMode.Clamp, null));
    fillBox.getCanvas().drawImageRect(cutImage, box, onto, blurPaint);
  }
  fillBox.getCanvas().drawImageRect(cutImage, box, onto, new CanvasKit.Paint());
  const fillImage = fillBox.makeImageSnapshot();

  const fillEffect = CanvasKit.RuntimeEffect.Make(HEAL_FILL_SKSL);
  assert.ok(fillEffect, 'HEAL_FILL_SKSL does not compile');
  const out = surface(W, H);
  const paint = new CanvasKit.Paint();
  paint.setShader(fillEffect.makeShaderWithChildren([], [asChild(src), asChildAt(fillImage, boxLeft, boxTop), asChild(mask)]));
  out.getCanvas().drawRect(CanvasKit.XYWHRect(0, 0, W, H), paint);
  return { pixels: pixelsOf(out.makeImageSnapshot()), mask: pixelsOf(mask), boxLeft, boxTop };
}

const flat = bytesOf(shadeFlat);
const made = healPass(flat, spots);
const alphaAt = (x, y) => made.mask[(y * W + x) * 4 + 3];
const inMask = (x, y) => alphaAt(x, y) > 200;
const rgbAt = (x, y) => {
  const i = (y * W + x) * 4;
  return [made.pixels[i], made.pixels[i + 1], made.pixels[i + 2]];
};

// 1. The mask is one shape. Painting a stroke as nine overlapping spots has to
//    leave one painted region, not nine: every painted pixel is reachable from
//    the first spot's centre without ever stepping off the mask. This is what
//    "mask liền mạch" is, and it is the thing the row of circles failed.
const painted = [];
for (let y = 0; y < H; y++) for (let x = 0; x < W; x++) if (inMask(x, y)) painted.push(y * W + x);
const seen = new Set();
const queue = [painted[0]];
seen.add(painted[0]);
while (queue.length) {
  const p = queue.pop();
  const x = p % W;
  const y = (p - x) / W;
  for (const [dx, dy] of [
    [1, 0],
    [-1, 0],
    [0, 1],
    [0, -1],
  ]) {
    const nx = x + dx;
    const ny = y + dy;
    if (nx < 0 || ny < 0 || nx >= W || ny >= H) continue;
    const q = ny * W + nx;
    if (seen.has(q) || !inMask(nx, ny)) continue;
    seen.add(q);
    queue.push(q);
  }
}
assert.equal(seen.size, painted.length, `${painted.length - seen.size} painted pixels are not reachable from the rest of the stroke`);
// ...and the mask is solid in the middle and clear well outside it.
assert.equal(alphaAt(Math.round(cx), Math.round(cy)), 255, 'the middle of the stroke is not fully painted');
assert.equal(alphaAt(2, 2), 0, 'the mask leaked into the corner of the frame');

// 2. The dust is gone and nothing was borrowed to do it: every painted pixel
//    ends on the ground's own value, 3.2 radii from a red card that a patch
//    search would have pasted in.
let worstIn = 0;
for (const p of painted) {
  const i = p * 4;
  for (let c = 0; c < 3; c++) worstIn = Math.max(worstIn, Math.abs(made.pixels[i + c] - GREY));
}
assert.ok(worstIn <= 4, `a painted pixel is off the ground it sits in by ${worstIn} levels (dust left, or a patch borrowed)`);

// 3. Nothing survives INSIDE the stroke either: neighbouring painted pixels
//    differ by a level or two, the way the ground does, where a spot solved on
//    its own would step at every seam.
let worstStep = 0;
for (let y = 0; y + 1 < H; y++) {
  for (let x = 0; x + 1 < W; x++) {
    if (!inMask(x, y) || !inMask(x + 1, y)) continue;
    worstStep = Math.max(worstStep, Math.abs(rgbAt(x, y)[0] - rgbAt(x + 1, y)[0]));
  }
}
assert.ok(worstStep <= 4, `the fill steps by ${worstStep} levels between neighbours inside the mask`);

// 4. The repair carries the LIGHT of where it landed. On a photo running from
//    dark to light across the frame, the painted pixels come back on that ramp
//    rather than at one flat value borrowed from somewhere else. The bound is
//    the ramp's own steepness, not zero: the fill reads the rim of the hole, and
//    where the hole takes weight off one side of a pixel its average slides that
//    way by about a pixel's worth of the ramp — 0.75 levels a pixel across this
//    frame. On the flat ground of 2 and 3 it is nothing at all.
const ramped = healPass(bytesOf(shadeRamp), spots);
let worstRamp = 0;
for (let y = 0; y < H; y++) {
  for (let x = 0; x < W; x++) {
    if (ramped.mask[(y * W + x) * 4 + 3] < 250) continue;
    worstRamp = Math.max(worstRamp, Math.abs(ramped.pixels[(y * W + x) * 4] - shadeRamp(x)[0]));
  }
}
assert.ok(worstRamp <= 6, `the fill is off the ramp it sits in by ${worstRamp} levels`);

// 5. Outside the mask, nothing moved. The pass is asked for the painted pixels
//    and no others — and "painted" is the whole brush, feather included: the last
//    15% of a spot is where the repair is mixed back in, so it is part of the
//    repair. Past the brush's own edge the photo is the photo.
let outWorst = 0;
let outAt = 0;
for (let p = 0; p < W * H; p++) {
  const x = p % W;
  const y = (p - x) / W;
  if (alphaAt(x, y) > 0) continue;
  for (let c = 0; c < 3; c++) {
    const off = Math.abs(made.pixels[p * 4 + c] - flat[p * 4 + c]);
    if (off > outWorst) {
      outWorst = off;
      outAt = x;
    }
  }
}
assert.equal(outWorst, 0, `a pixel outside the mask (x=${outAt}) changed by ${outWorst} levels`);

// 6. The same photo and the same spots give the same photo, byte for byte.
assert.deepEqual(
  Buffer.from(healPass(flat, spots).pixels),
  Buffer.from(made.pixels),
  'the pass is not deterministic'
);

// 7. A mask with no rim to read from — one spot over the whole frame — leaves
//    the photo alone. Nothing is painted black, because nothing was found to
//    paint it with, and a frame is still a frame.
const all = healPass(flat, [{ x: 0.5, y: 0.5, r: 2 }]);
let worstAll = 0;
for (let p = 0; p < W * H; p++) {
  for (let c = 0; c < 3; c++) worstAll = Math.max(worstAll, Math.abs(all.pixels[p * 4 + c] - flat[p * 4 + c]));
}
assert.equal(worstAll, 0, `a frame-wide mask changed the photo by ${worstAll} levels`);

// 8. The shape the brush is actually used in: one long thin run right across the
//    frame, on a photo that runs dark to light along it. Its rim is the pixels
//    just above and below it wherever it passes, so the repair comes back on the
//    ramp there — not at the frame's average colour. A hole this long needs the
//    level count to grow with it: too few levels and the middle of the run is
//    farther from any rim than the widest blur reaches, so the fill comes back
//    the average of a blur too wide to have read the rim — one flat mark.
const band = [];
for (let i = 0; i <= 60; i++) band.push({ x: 0.12 + (0.76 * i) / 60, y: 0.5, r: 0.02 });
const across = healPass(bytesOf(shadeRamp), band);
let worstBand = 0;
let bandAt = 0;
for (let y = 0; y < H; y++) {
  for (let x = 0; x < W; x++) {
    if (across.mask[(y * W + x) * 4 + 3] < 250) continue;
    const off = Math.abs(across.pixels[(y * W + x) * 4] - shadeRamp(x)[0]);
    if (off > worstBand) {
      worstBand = off;
      bandAt = x;
    }
  }
}
assert.ok(worstBand <= 8, `a painted pixel at x=${bandAt} is ${worstBand} levels off the rim it sits in`);

// 9. The one case every size above hides: a frame big enough that the fill's box
//    sits INSIDE it. On a 160px frame with a 8px brush the padding is wider than
//    the frame, so the box is the frame and its corner is (0,0) — a fill drawn at
//    the box's coordinates and one drawn at the surface's own are then the same
//    draw, and a repair off by the box's corner is off by nothing. On a real photo
//    it is not nothing: a brush at the sensor-dust end of the scale, a frame a few
//    thousand pixels across, and the corner is hundreds of pixels up and to the
//    left of the spot. That is the shape the reader heals with, and on a ramp the
//    patch comes back at the ramp's value hundreds of pixels away — the healed
//    area is the wrong shade, which is what this case is here to catch.
{
  const keptW = W;
  const keptH = H;
  // Tall enough that the box clears BOTH edges: the padding is three blur
  // radii of the widest brush, which on this frame is 192px, so a shorter
  // frame would push boxTop to 16 and the corner would nearly be the edge.
  W = 640;
  H = 720;
  const shadeWide = (x) => {
    const v = 40 + (120 * x) / (W - 1);
    return [v, v, v];
  };
  const spot = [{ x: 0.5, y: 0.5, r: 0.05 }]; // a 32px brush on this frame
  const wide = healPass(bytesOf(shadeWide), spot);
  assert.ok(
    wide.boxLeft > 16 && wide.boxTop > 16,
    `the fill's box is not inset on this frame (boxLeft=${wide.boxLeft}, boxTop=${wide.boxTop}) — this case is no longer testing the bug it exists for`
  );
  let worstWide = 0;
  let wideAt = 0;
  for (let y = 0; y < H; y++) {
    for (let x = 0; x < W; x++) {
      if (wide.mask[(y * W + x) * 4 + 3] < 250) continue;
      const off = Math.abs(wide.pixels[(y * W + x) * 4] - shadeWide(x)[0]);
      if (off > worstWide) {
        worstWide = off;
        wideAt = x;
      }
    }
  }
  assert.ok(
    worstWide <= 4,
    `a spot healed at the middle of a ${W}x${H} frame came back ${worstWide} levels off at x=${wideAt} — the fill is read at the box's corner, not at the spot (box is at ${wide.boxLeft},${wide.boxTop})`
  );
  W = keptW;
  H = keptH;
}

console.log('ok  the stroke is one mask, the fill carries the light, and nothing outside it moved');