RecipesCam/docker/frontend/src/engine/rawDevelop.ts

// RAW → JPEG on the client, so a camera's own file opens in the studio without
// a DNG converter in the middle (see native_raw_processing_opfs_architecture.md).
//
// A RAW opens at the colour the camera chose for it, and at the sensor's own
// resolution. LibRaw is kept out of white balance and tone (below) and knows
// nothing of the body's picture style; the one rendering the file does carry with
// it is the preview the camera wrote inside it — the frame the photographer saw
// and the one a desktop viewer shows.
//
// The develop is carried by the difference between that preview and this develop's
// own rendering of the same frame. The difference is kept at the preview's own
// resolution, one byte a channel with 0.5 for a pixel the two already agree on,
// and added back on the value the develop leaves: the frame opens at the camera's
// colour and at the sensor's resolution, and the develop is only ever asked for
// the detail the preview never had. Nothing is fitted — a body is matched against
// its own preview the moment it opens, whatever its picture style is, and there is
// no table to go stale (see previewResidual below).
//
// It replaced a per-channel cubic fitted between a grid of each, and a 3x3 least
// squared ahead of that curve before it, neither of which is the camera's
// rendering: a picture style is a colour transform with a curve in it. Measured
// against the file's own preview, mean dE00: 6.04 for the cubic on the Fujifilm
// RAF this was reported on, 0.54 for the residual on that frame and 0.00 on every
// body whose preview is the size of its sensor (a Nikon NEF, a DNG), where the two
// frames agree block for block and the residual is nothing.
//
// What the residual gives up is the headroom: it lands the frame on the preview's
// own white, so the two stops the sensor holds above white are not kept (the cubic
// kept 2.3% of the frame at pure white against 3.3% in the preview itself). A
// highlight recovery that needs them has to come from the develop, not from here.
//
// The preview is also the fallback, not the frame: it is what a file with no
// usable preview (some DNG), or with sensor data that will not decode, opens as.
//
// The develop: LibRaw demosaics in its own worker; what comes back is linear camera data,
// which this file turns into the sRGB the rest of the pipeline expects. The
// settings below were checked against the preview a Sony ILME-FX30 writes into
// its own ARW (the camera JPEG, read straight out of the file): the developed
// frame and that preview agree to within 1% on both channel ratios, R/G 0.909
// against 0.903 and B/G 0.603 against 0.606.
//   - noAutoScale + useCameraWb:false + noAutoBright + gamm [1,1] keep LibRaw out
//     of white balance and tone, so `cam_mul` and `rgb_cam` can be applied here
//     exactly once.
//   - halfSize:false: the frame opens at the sensor's own resolution — a 24MP
//     RAW develops to 24MP (6020x4024 on the GR, 6000x4000 on the Fuji), not to
//     the quarter the half-size demosaic reports. `userQual` 3 then demosaics
//     all of it. The develop was half-size until a GR's DNG opened at 3010x2012
//     against its own JPEG's 6000x4000: the quarter-size frame was the flag,
//     not the file. ponytail: costs ~4x the develop time and two full-size F16
//     surfaces; put the flag behind a "draft" toggle if a phone ever has to.
//
// The band loop exists because a single Float32 copy of the whole plane would be
// ~100MB. Each band is decoded, normalised and drawn before the next is read.
// ponytail: the develop itself is one full-resolution shader draw per band, on
// the main thread (Skia is not available in the RAW worker). Move it to a worker
// with an OffscreenCanvas if the develop ever blocks the UI visibly.
import LibRaw from 'libraw-wasm';
import { sensorWhite } from './sensorWhite';
import { f32ToF16 } from './halfFloat';
import { MATCH_GRID } from './previewMatch';
import { Skia, initSkia } from './skiaShim';
import { decodeTiff } from './tiffDecode';

// What `imageData()` returns for the settings below: 16-bit, 3 channels, black
// level already gone — the post-process subtracts it whatever `noAutoScale`
// says, which only holds back the white balance and the output scaling.
//
// Its white level is not `maximum` but the frame's own plateau, a factor of 1.89
// to 2.00 out (see sensorWhite below). The gain carries that level, so the white
// lands back on 1.0 — and on every body, not just the two that factor two was
// fitted on: without it every frame opened a stop bright (the FX30's own JPEG has
// 0.05% of pixels at pure white where the develop had 0.76%) and a body that sat
// lower (X-Trans, 1.892) never even reached the highlight desaturation, which
// starts at 0.95 of the sensor.
const SETTINGS = {
  halfSize: false,
  outputBps: 16,
  outputColor: 0,
  noAutoScale: true,
  useCameraWb: false,
  noAutoBright: true,
  gamm: [1, 1] as [number, number],
  userQual: 3,
  highlight: 2,
};

// 16-bit samples, but everything below is [0,1] — Skia reads an F32 image as
// already-normalised colour, and a raw 0..65535 plane comes back black.
const SAMPLE_MAX = 65535;
// Pixels per band, held under the point where the Float32 copy dominates the
// memory the page is allowed to use.
const BAND_PIXELS = 2_000_000;

const RAW_DEVELOP_SKSL = `
uniform shader raw;
uniform float gain;     // 1 / the white level the frame itself ran out at
uniform float4 mul;     // cam_mul, green-normalised
uniform float4 m0;      // camera -> sRGB, the first three columns of rgb_cam
uniform float4 m1;
uniform float4 m2;
uniform float4 crop;    // (y offset of this band, 0, 0, 0)
uniform shader res;     // the file's own rendering, as a residual (see below)
uniform float2 rscl;    // frame pixels -> residual pixels

float3 encode(float3 x) {
  x = clamp(x, 0.0, 1.0);
  return mix(x * 12.92, 1.055 * pow(x, float3(1.0 / 2.4)) - 0.055, step(float3(0.0031308), x));
}

half4 main(float2 pos) {
  float4 p = raw.eval(float2(pos.x, pos.y - crop.x));
  // The plane arrives with the black level already subtracted — it floors at 0,
  // not at color_data.black (measured on the FX30 ARW: the sensor mosaic floors
  // at 334, the plane at 0, a quarter of its red samples under the black level).
  // Subtracting it again drained red and blue — the two channels the gains lift
  // most — and dragged every frame towards green.
  float3 n = max(p.rgb * gain, 0.0);   // the sensor's own levels, white level 1.0
  float3 lin = n * mul.rgb;
  // Only the floor there. A photo the sensor could not hold goes over the white
  // level in all three channels, and clipping them one by one before the WB gains
  // is what tints what is left of the highlight: green — the channel the gains are
  // normalised to — stops at 1.0 while red and blue, which need their 2.6x and
  // 1.6x, are already past it, so the blown area comes out magenta. Keep the
  // channel ratios through the matrix instead and let the overflow fade to white.
  float3 rgb = float3(dot(m0.xyz, lin), dot(m1.xyz, lin), dot(m2.xyz, lin));
  float mx = max(max(rgb.r, rgb.g), rgb.b);
  // A pixel that has run to the white level has no colour of its own left to keep,
  // and what the gains made of it is an artefact, not a colour: ease the pixel
  // towards the neutral of its own value as that point is approached. The clip to
  // read is the one the gains make, not only the sensor's own — the gains here are
  // 1.7x and 1.9x on red and blue, so a blown sky reaches the white level at 0.59 of
  // the raw range in those channels while the green, which the gains are
  // normalised to, only reaches it at 1.0. Read on the sensor's levels alone the
  // gate stayed shut across a whole blown sky and left the develop's magenta in it
  // (227,184,245 at the gate's own value against 245,245,245 read where the gains
  // put the clip, the camera's preview white at that block). Both are read, so a
  // body whose gains do not lift a channel keeps the sensor's own clip as its gate.
  // mx is the value to hold: it is the pixel's own lightness.
  float hi = max(max(n.r, n.g), n.b);
  hi = max(hi, max(max(lin.r, lin.g), lin.b));
  float blown = smoothstep(0.82, 0.98, hi);
  rgb = mix(rgb, float3(mx), blown);
  // The overflow used to fade towards white — mix(rgb / mx, 1, 1 - 1 / mx) —
  // which put every pixel of a blown sky on exactly 1.0 and threw the two stops
  // the sensor held above white away with it: LIGHT's HIGHLIGHT row then had a
  // flat white to pull on and nothing to reveal. The white point is moved down
  // instead and the overflow squeezed back in under it by the doc's soft knee:
  //
  //   y = T + over / (1 + 2S*over),  over = mx - T,  S = 1 / (2(1 - T))
  //
  // S is what puts 1.0 on the asymptote, so the sensor's own plateau — two white
  // levels up, see the gain above — lands at ~0.94 and the first stop over white
  // spends 0.85..0.94. Below T the frame is untouched and the curve leaves T with
  // the slope it arrived with (1), so there is no seam to mask; above it the frame
  // darkens, which is the one move no later pass can undo — which is the point,
  // and what the per-file curve below then measures the REST of the frame back from.
  //
  // ponytail: 4.7 stops of headroom now share ~5% of the ramp, and the develop
  // still leaves as an 8-bit JPEG. Give it a float16 output when RAW highlights
  // have to print rather than merely be seen.
  if (mx > 0.7) {
    float over = mx - 0.7;
    rgb *= (0.7 + over / (1.0 + over * 3.3333)) / mx;
  }
  float3 e = encode(rgb);
  // The neutral the gate above drew, kept on the way out.
  float3 o = mix(e, float3(max(max(e.r, e.g), e.b)), blown);
  // The file's own colour and tone, carried onto the develop at the develop's own
  // resolution. res is the camera's rendering less this develop's own low
  // frequencies, one byte per channel with 0.5 as nothing-to-say, so adding it
  // puts the camera's rendering back on the frame and leaves everything the
  // develop has that the preview never did — the sensor's own detail — in the
  // frame. pos is already the frame's own coordinate (the band is drawn at its
  // own y offset, so the same pos is band-local only because crop.x is taken back
  // off for the raw tap above), so one scale is the whole mapping.
  float2 rp = pos * rscl;
  o = clamp(o + (res.eval(rp).rgb - 0.5), 0.0, 1.0);
  return half4(half3(o), 1.0);
}
`;

// The residual the develop above is carried by: the camera's own frame less the
// same frame as the develop left it, one byte per channel, 0.5 for a pixel the
// two already agree on. Both children are drawn at the size the residual is kept
// at, so neither needs a local matrix, and a minus b has to be able to go
// negative — hence the 0.5, which the develop takes back off.
const RESIDUAL_SKSL = `
uniform shader dev;
uniform shader ref;

half4 main(float2 p) {
  return half4(half3(clamp(ref.eval(p).rgb - dev.eval(p).rgb + 0.5, 0.0, 1.0)), 1.0);
}
`;

// The extensions LibRaw is compiled for. Not `tif`: a TIFF in a camera folder is
// as likely to be an export as a RAW, and opening one asked for PRO and wore a
// RAW badge for a file the engine has nothing to develop in. It is read for its
// own pixels instead — see `tiffDevelop`.
const RAW_EXT = [
  'arw', 'srf', 'sr2', 'cr2', 'cr3', 'crw', 'nef', 'nrw', 'dng', 'orf', 'raf',
  'rw2', 'pef', 'srw', 'raw', 'rwl', '3fr', 'fff', 'iiq', 'mos', 'mrw', 'x3f',
];

export const RAW_ACCEPT = RAW_EXT.map((e) => `.${e}`).join(',');

export function isRawName(name: string): boolean {
  const ext = name.slice(name.lastIndexOf('.') + 1).toLowerCase();
  return name.includes('.') && RAW_EXT.includes(ext);
}

// A TIFF is not a RAW: it is a picture no decoder here draws by itself, and the
// one test for it, so the tile's route and the studio's route cannot drift apart.
export function isTiffName(name: string): boolean {
  return /\.(tiff?)$/i.test(name);
}

// The camera's own preview, when the file carries one: the colour reference the
// curve below is fitted to, and what the file opens as when the sensor does not
// decode.
// ponytail: it is 1616x1080 on an A5100 and 1620x1080 on an FX30, so handing it
// back is a 1.7MP frame — a print past it has to come off the develop, which is
// what a developed file already gives.
async function cameraPreview(raw: LibRaw): Promise<Uint8Array | null> {
  const thumb = await raw.thumbnailData().catch(() => undefined);
  if (thumb?.format !== 'jpeg' || !thumb.data?.length) return null;
  return new Uint8Array(thumb.data);
}

// Fast pure JS binary scanner to extract embedded JPEG preview from RAW files (ARW, NEF, CR2, CR3, DNG, RAF, RW2, ORF, PEF).
// Scans for all embedded JPEGs in the RAW file and returns the largest segment (the full-size camera preview).
export function extractEmbeddedJpeg(buf: Uint8Array): Uint8Array | null {
  const len = buf.length;
  let largest: Uint8Array | null = null;
  let maxLen = 0;

  for (let i = 0; i < len - 4; i++) {
    if (buf[i] === 0xff && buf[i + 1] === 0xd8 && buf[i + 2] === 0xff) {
      const start = i;
      let pos = i + 2;
      let eoiPos = -1;
      let validJpeg = true;

      while (pos < len - 4) {
        if (buf[pos] !== 0xff) {
          validJpeg = false;
          break;
        }
        const m = buf[pos + 1];
        if (m === 0xd9) {
          eoiPos = pos + 2;
          break;
        }
        if (m === 0xda) {
          const sosHeaderLen = (buf[pos + 2] << 8) | buf[pos + 3];
          if (sosHeaderLen < 2 || pos + 2 + sosHeaderLen > len) {
            validJpeg = false;
            break;
          }
          let scanPos = pos + 2 + sosHeaderLen;
          const searchEnd = Math.min(len - 1, scanPos + 20 * 1024 * 1024);
          while (scanPos < searchEnd) {
            if (buf[scanPos] === 0xff) {
              const marker = buf[scanPos + 1];
              if (marker === 0xd9) {
                eoiPos = scanPos + 2;
                break;
              }
              if (marker === 0x00 || (marker >= 0xd0 && marker <= 0xd7)) {
                scanPos += 2;
                continue;
              }
            }
            scanPos++;
          }
          break;
        }
        if (m === 0x00 || (m >= 0xd0 && m <= 0xd7)) {
          pos += 2;
          continue;
        }
        const segLen = (buf[pos + 2] << 8) | buf[pos + 3];
        if (segLen < 2 || pos + 2 + segLen > len) {
          validJpeg = false;
          break;
        }
        pos += 2 + segLen;
      }

      if (eoiPos > start && validJpeg) {
        const segSize = eoiPos - start;
        if (segSize > 10000 && segSize > maxLen) {
          maxLen = segSize;
          largest = buf.subarray(start, eoiPos);
        }
        i = eoiPos - 1;
      }
    }
  }
  return largest;
}

// The preview on its own, for the catalogue: a folder of RAW files has to show
// a tile per frame, and unpack_thumb is a seek and a copy where the develop
// above is a full decode of every pixel at full resolution. No preview inside
// the file means no tile — the row still lists the frame by name, and the
// studio develops it the moment it is opened.
//
// What CanvasKit draws, and only that: a TIFF goes to `decodeTiff` below, since
// `MakeImageFromEncoded` answers null to one (measured on a baseline 5472x3648
// scan) and the file has to be read by hand.
function tiffImage(bytes: Uint8Array, maxSide: number): any {
  const img = Skia.Image.MakeImageFromEncoded(bytes);
  if (img) return img;

  const frame = decodeTiff(bytes, maxSide);
  if (!frame) return null;
  return Skia.Image.MakeImage(
    {
      width: frame.width,
      height: frame.height,
      colorType: Skia.ColorType.RGBA_8888,
      alphaType: Skia.AlphaType.Unpremul,
      colorSpace: Skia.ColorSpace.SRGB,
    },
    frame.rgba,
    frame.width * 4
  );
}

// The same read, encoded to JPEG. `max` is the long side it comes out at, 0 for
// the file's own size.
function tiffJpeg(bytes: Uint8Array, max: number): Uint8Array | null {
  try {
    const img = tiffImage(bytes, max || Infinity);
    if (!img) return null;
    try {
      return shrinkToJpeg(img, max || Math.max(img.width(), img.height()));
    } finally {
      img.delete();
    }
  } catch {
    return null;
  }
}

export function tiffThumbnail(bytes: Uint8Array): Uint8Array | null {
  return tiffJpeg(bytes, 640);
}

// A `.tif` read for the screen, now that it is no file the engine develops and
// no decoder the browser has: the frame is the picture, so it is decoded here —
// by the studio at the file's own size, by the catalogue's stage at what a screen
// shows (`max`, 0 for the file's own). CanvasKit is what draws it, and neither
// caller has booted it: a visitor who only ever opens the wall never does.
//
// ponytail: at the file's own size this is the frame's RGBA in memory (80MB for a
// baseline 5472x3648 scan) plus the canvas it is drawn to. Cap it if a plate ever
// arrives that does not fit; the develop above is the same order already.
export async function tiffDevelop(bytes: Uint8Array, max = 0): Promise<Uint8Array | null> {
  if (!Skia) await initSkia({ locateFile: (file) => `/wasm/${file}` });
  return tiffJpeg(bytes, max);
}

// A frame shrunk to the square the wall paints and encoded to JPEG — the one
// step of a tile that is CanvasKit's whatever the frame came out of.
function shrinkToJpeg(img: any, n = 640): Uint8Array | null {
  const w = img.width();
  const h = img.height();
  if (!w || !h) return null;
  const wide = w >= h;
  const targetW = wide ? n : Math.max(1, Math.round((w / h) * n));
  const targetH = wide ? Math.max(1, Math.round((h / w) * n)) : n;

  const surface = Skia.Surface.MakeOffscreen(targetW, targetH) ?? Skia.Surface.Make(targetW, targetH);
  if (!surface) return null;
  const canvas = surface.getCanvas();
  canvas.drawImageRectCubic(
    img,
    Skia.XYWHRect(0, 0, w, h),
    Skia.XYWHRect(0, 0, targetW, targetH),
    1 / 3,
    1 / 3
  );
  surface.flush();
  const snapshot = surface.makeImageSnapshot();
  surface.dispose();
  if (!snapshot) return null;
  const jpeg = snapshot.encodeToBytes(Skia.ImageFormat.JPEG, 80);
  snapshot.dispose();
  return jpeg ? new Uint8Array(jpeg) : null;
}

// The preview on its own, for the catalogue: a folder of RAW files has to show
// a tile per frame, and unpack_thumb is a seek and a copy where the develop
// above is a full decode of every pixel at full resolution. No preview inside
// the file means no tile — the row still lists the frame by name, and the
// studio develops it the moment it is opened.
export async function rawThumbnail(bytes: Uint8Array, fileName?: string): Promise<Uint8Array | null> {
  // A `.tif` is not a RAW: the file IS the picture, and no camera preview is
  // hiding in it — scanning the bytes for one finds the frame's own pixel data
  // and hands back three hundred kilobytes of it dressed as a JPEG, which is
  // what the wall used to keep as the frame's tile and draw as nothing.
  if (fileName && isTiffName(fileName)) {
    // CanvasKit is what draws the JPEG inside one, and the catalogue has a route
    // of its own: the studio boots the engine, a visitor who only ever opens the
    // wall never does, and a tile read before it is up draws as nothing.
    if (!Skia) await initSkia({ locateFile: (file) => `/wasm/${file}` });
    return tiffThumbnail(bytes);
  }

  const fast = extractEmbeddedJpeg(bytes);
  if (fast) return fast;

  const raw = new LibRaw();
  try {
    // The same transfer as in the develop: the worker is handed a copy, so the
    // caller's bytes stay whole for whatever the catalogue reads off them next.
    await raw.open(bytes.slice() as unknown as BufferSource, SETTINGS);
    return await cameraPreview(raw);
  } catch {
    return tiffThumbnail(bytes);
  } finally {
    try {
      raw.dispose();
    } catch {}
  }
}

// MATCH_GRID x MATCH_GRID block colours of a frame, one byte per channel.
function gridOf(image: any, n = MATCH_GRID): Uint8Array | null {
  const surface = Skia.Surface.MakeOffscreen(n, n) ?? Skia.Surface.Make(n, n);
  if (!surface) return null;
  const canvas = surface.getCanvas();
  // Cubic, not a linear tap: this is a 45x reduction and linear reads a handful
  // of source pixels per block — noise for the curve to be fitted on.
  canvas.drawImageRectCubic(
    image,
    Skia.XYWHRect(0, 0, image.width(), image.height()),
    Skia.XYWHRect(0, 0, n, n),
    1 / 3,
    1 / 3
  );
  surface.flush();
  const px = canvas.readPixels(0, 0, {
    width: n,
    height: n,
    colorType: Skia.ColorType.RGBA_8888,
    alphaType: Skia.AlphaType.Unpremul,
    colorSpace: Skia.ColorSpace.SRGB,
  }) as Uint8Array | null;
  surface.dispose();
  return px ? new Uint8Array(px.buffer, px.byteOffset, px.byteLength) : null;
}

// The same grid out of the preview, which the file carries as a JPEG. Decoded and
// reduced through the same Skia call as the develop's own grid, because the two
// grids are only comparable — and a curve between them only meaningful — when one
// resampler made both. A 2D canvas here instead left the curve following its own
// smoothing: on the A5100 frame the same develop scored dE00 4.7 against 4.4, and
// the dark end of the frame came out 6 L further from the preview than the curve
// it was handed asked for.
function previewGrid(jpeg: Uint8Array, w: number, h: number, n = MATCH_GRID): Uint8Array | null {
  const bmp = Skia.Image.MakeImageFromEncoded(jpeg);
  if (!bmp) return null;
  try {
    if (bmp.width() < n || bmp.height() < n) return null;
    // The decode above has already turned the preview by its own EXIF
    // orientation: a 1920x1280 orientation-8 Panasonic preview decodes to
    // 1280x1920, the frame the camera meant. So these are display dimensions and
    // the only thing left is to trim them to the develop's aspect. Rotating here
    // as well — which this did — turned a portrait frame's reference a quarter
    // turn against the develop it was being fitted to, and the fit was rejected
    // as folded (see previewMatch), which is why the Panasonic RW2 opened with no
    // curve at all.
    const { x: cropX, y: cropY, w: cropW, h: cropH } = previewCrop(bmp, w, h);

    const surface = Skia.Surface.MakeOffscreen(n, n) ?? Skia.Surface.Make(n, n);
    if (!surface) return null;
    const canvas = surface.getCanvas();
    canvas.drawImageRectCubic(
      bmp,
      Skia.XYWHRect(cropX, cropY, cropW, cropH),
      Skia.XYWHRect(0, 0, n, n),
      1 / 3,
      1 / 3
    );
    surface.flush();
    const px = canvas.readPixels(0, 0, {
      width: n,
      height: n,
      colorType: Skia.ColorType.RGBA_8888,
      alphaType: Skia.AlphaType.Unpremul,
      colorSpace: Skia.ColorSpace.SRGB,
    }) as Uint8Array | null;
    surface.dispose();
    return px ? new Uint8Array(px.buffer, px.byteOffset, px.byteLength) : null;
  } finally {
    bmp.delete();
  }
}

// The preview trimmed to the develop's own aspect, centred. What a body writes
// into the file is the frame the photographer saw, and it is the frame the
// sensor read; anything left over is a margin the develop does not have.
function previewCrop(bmp: any, w: number, h: number): { x: number; y: number; w: number; h: number } {
  const aspect = w / h;
  let cw = bmp.width();
  let ch = bmp.height();
  if (Math.abs(cw / ch - aspect) > 0.01) {
    if (cw / ch > aspect) cw = Math.round(ch * aspect);
    else ch = Math.round(cw / aspect);
  }
  return { x: Math.round((bmp.width() - cw) / 2), y: Math.round((bmp.height() - ch) / 2), w: cw, h: ch };
}

// The residual the develop is carried by, and how many frame pixels one of its
// own is worth.
export interface PreviewResidual {
  img: any;
  sx: number;
  sy: number;
}

// The residual's own long side. It is a smooth map by construction — it holds no
// detail, that is the develop's half of the frame — so a preview wider than this
// is reduced rather than carried whole: the colour it carries survives the
// reduction, and the 24MP surface a NEF-sized preview would need does not.
const RESIDUAL_MAX = 2048;

// The one residual that says nothing: a pixel the camera's frame and this develop
// already agree on, for a file whose preview gave nothing to carry.
let NO_RESIDUAL: any = null;
function noResidual(): any {
  if (!NO_RESIDUAL) {
    // 0.5 in half, alpha 1 — the value the develop takes 0.5 back off.
    const half = new Uint16Array([0x3800, 0x3800, 0x3800, 0x3c00]);
    NO_RESIDUAL = Skia.Image.MakeImage(
      { width: 1, height: 1, colorType: Skia.ColorType.RGBA_F16, alphaType: Skia.AlphaType.Unpremul },
      new Uint8Array(half.buffer),
      8
    );
  }
  return NO_RESIDUAL;
}

// What the camera's own rendering has to say about this develop: the whole
// difference between the two frames — the body's white balance, picture style,
// tone curve and colour, whatever the body is — carried at the preview's
// resolution as one byte a channel, 0.5 where the two already agree. The develop
// adds it back on the value it left, so the frame opens at the colour the
// photographer saw AND at the sensor's own resolution: the develop is only ever
// asked for the detail the preview never had.
//
// A mapping was tried here first — one cubic a channel, and a 3x3 under it before
// that — and neither is the camera's rendering: a picture style is a colour
// transform with a curve in it, and the best a per-channel cubic fitted between
// two grids could reach on the Fujifilm RAF this was reported on was dE00 6.04
// against the file's own preview. The residual reaches 0.54 on the same frame and
// 0.00 on every body that writes a full-size preview, because it is not a fit at
// all — it is the difference itself.
function previewResidual(first: any, jpeg: Uint8Array, w: number, h: number): PreviewResidual | null {
  const bmp = Skia.Image.MakeImageFromEncoded(jpeg);
  if (!bmp) return null;
  let ref: any = null;
  let dev: any = null;
  let refShader: any = null;
  let devShader: any = null;
  let resShader: any = null;
  let paint: any = null;
  try {
    const crop = previewCrop(bmp, w, h);
    const cap = Math.min(1, RESIDUAL_MAX / Math.max(crop.w, crop.h));
    const rw = Math.max(1, Math.round(crop.w * cap));
    const rh = Math.max(1, Math.round(crop.h * cap));

    // One small surface per frame, both at the residual's own size, so neither
    // child needs a local matrix and the subtraction below is one tap for one tap.
    const small = (draw: (canvas: any) => void): any => {
      const surface = Skia.Surface.MakeOffscreen(rw, rh) ?? Skia.Surface.Make(rw, rh);
      if (!surface) return null;
      draw(surface.getCanvas());
      surface.flush();
      const shot = surface.makeImageSnapshot();
      surface.dispose();
      return shot;
    };

    ref = small((canvas) =>
      canvas.drawImageRectCubic(
        bmp,
        Skia.XYWHRect(crop.x, crop.y, crop.w, crop.h),
        Skia.XYWHRect(0, 0, rw, rh),
        1 / 3,
        1 / 3
      )
    );
    dev = small((canvas) =>
      canvas.drawImageRectCubic(
        first,
        Skia.XYWHRect(0, 0, w, h),
        Skia.XYWHRect(0, 0, rw, rh),
        1 / 3,
        1 / 3
      )
    );
    if (!ref || !dev) return null;

    const effect = Skia.RuntimeEffect.Make(RESIDUAL_SKSL);
    if (!effect) return null;
    const tap = (img: any) =>
      img.makeShaderOptions(
        Skia.TileMode.Clamp,
        Skia.TileMode.Clamp,
        Skia.FilterMode.Nearest,
        Skia.MipmapMode.None
      );
    devShader = tap(dev);
    refShader = tap(ref);
    resShader = effect.makeShaderWithChildren(new Float32Array(0), [devShader, refShader]);
    paint = Skia.Paint();
    paint.setShader(resShader);
    const out = small((canvas) => canvas.drawRect(Skia.XYWHRect(0, 0, rw, rh), paint));
    // The residual is read back at the develop's own resolution by a bilinear
    // tap, so it is kept smooth: one byte a channel is finer than the shading
    // it carries, which is the whole difference between two renderings.
    return out ? { img: out, sx: rw / w, sy: rh / h } : null;
  } catch {
    return null;
  } finally {
    bmp.delete();
    ref?.delete();
    dev?.delete();
    refShader?.delete();
    devShader?.delete();
    resShader?.delete();
    paint?.delete();
  }
}

// The camera -> sRGB matrix, applied to the WB'd sensor triple.
//
// `rgb_cam` is dcraw's own: it is the camera -> sRGB transform already, its rows
// sum to one, and the 1/num dcraw scales it by rides in `pre_mul`. Dividing it by
// pre_mul a second time undoes the normalisation and drags the frame towards
// green. It is what every body LibRaw knows a `cam_xyz` for gets — the Olympus
// ORF this was reported on included, whose `rgb_cam` and `cam_xyz` disagree
// (measured: rgb_cam x cam_mul lands on R/G 1.021 B/G 0.920 against the file's own
// preview's 1.013 / 0.841, the cam_xyz chain this used to prefer on 0.921 / 0.886
// — green, and blue for a scene the camera left neutral).
//
// `cam_xyz` stays as the fallback for a file that has one and no `rgb_cam`: its
// rows are the XYZ of each camera channel, so sRGB = xyz_rgb^-1 . cam_xyz^T . cam,
// with the rows normalised so a neutral camera triple opens neutral, as dcraw's
// cam_xyz_coeff normalises them.
const XYZ_TO_SRGB = [
  [3.2404542, -1.5371385, -0.4985314],
  [-0.9692660, 1.8760108, 0.0415560],
  [0.0556434, -0.2040259, 1.0572252],
];

export type CameraBrand =
  | 'fujifilm'
  | 'sony'
  | 'canon'
  | 'nikon'
  | 'olympus'
  | 'panasonic'
  | 'leica'
  | 'apple'
  | 'ricoh'
  | 'dng'
  | 'generic';

export function detectCameraBrand(fileName?: string, meta?: any): CameraBrand {
  const ext = fileName ? fileName.slice(fileName.lastIndexOf('.') + 1).toLowerCase() : '';
  const make = (meta?.idata?.make || meta?.make || '').toLowerCase();

  if (ext === 'raf' || make.includes('fuji')) return 'fujifilm';
  if (ext === 'arw' || ext === 'srf' || ext === 'sr2' || make.includes('sony')) return 'sony';
  if (ext === 'cr2' || ext === 'cr3' || ext === 'crw' || make.includes('canon')) return 'canon';
  if (ext === 'nef' || ext === 'nrw' || make.includes('nikon')) return 'nikon';
  if (ext === 'orf' || make.includes('olympus') || make.includes('om digital')) return 'olympus';
  if (ext === 'rw2' || make.includes('panasonic') || make.includes('lumix')) return 'panasonic';
  if (ext === 'pef' || make.includes('ricoh') || make.includes('pentax')) return 'ricoh';
  if (make.includes('leica')) return 'leica';
  if (make.includes('apple') || make.includes('iphone')) return 'apple';
  if (ext === 'dng') return 'dng';
  return 'generic';
}

export function getBrandWbMultipliers(brand: CameraBrand, cd: any): number[] {
  // 1. The balance the body recorded with the frame. `pre_mul` is the daylight one
  // LibRaw carries for the model, and it is what used to be read first — on the
  // Fujifilm RAF this was reported on it is the wrong one twice over (R/G 2.296
  // against the frame's own 1.795, B/G 1.307 against 1.626), which left blue short
  // and every white in the frame pink. Measured against the file's own preview over
  // the sample set, dE00 6.73 -> 5.27 on that frame and 7.52 -> 7.26 on the X-T20's.
  if (cd?.cam_mul && Array.isArray(cd.cam_mul) && cd.cam_mul.some((v: number) => v > 0 && v !== 1)) {
    return cd.cam_mul;
  }

  // 2. Otherwise the daylight balance LibRaw worked out for the model.
  if (cd?.pre_mul && Array.isArray(cd.pre_mul) && cd.pre_mul.some((v: number) => v > 0 && v !== 1)) {
    return cd.pre_mul;
  }

  // 3. Brand-calibrated WB fallbacks (prevents generic single fallback from producing color casts)
  switch (brand) {
    case 'fujifilm':
      return [2.24, 1.0, 1.72, 1.0];
    case 'sony':
      return [1.95, 1.0, 1.55, 1.0];
    case 'canon':
      return [2.10, 1.0, 1.48, 1.0];
    case 'nikon':
      return [1.88, 1.0, 1.42, 1.0];
    case 'olympus':
      return [2.15, 1.0, 1.38, 1.0];
    case 'panasonic':
      return [1.98, 1.0, 1.45, 1.0];
    case 'leica':
      return [1.85, 1.0, 1.65, 1.0];
    case 'apple':
      return [2.05, 1.0, 1.82, 1.0];
    case 'ricoh':
      return [1.96, 1.0, 1.52, 1.0];
    case 'dng':
    default:
      return [2.00, 1.0, 1.50, 1.0];
  }
}

// The balance the file's own preview was rendered with.
//
// Neither balance LibRaw hands over is it. `cam_mul` is what the body recorded and
// `pre_mul` the daylight one it carries for the model, and the preview in the file
// is the rendering the body actually made — so the balance is in there, read off the
// blocks the camera left bright and near neutral: average the sensor triple behind
// each of those blocks over the block's own pixels, and the triple that makes those
// blocks come out equal is the frame's white. The camera's rendering is the same
// reference the tone curve below is fitted to; this is its WB row.
//
// Read on the frame, not on the file's tags, which is why it holds across bodies:
// measured against the embedded preview, the Fujifilm RAF this was reported on came
// back at block a* -1.9 against the preview's -2.1 where `pre_mul` gave 0.0 and
// `cam_mul` -1.0 — the pink in every white of the frame — and dE00 4.93 where they
// gave 6.73 and 5.27. On the X-T20 frame: -4.2 against -3.9, dE00 6.56 against 7.52
// and 7.26. Both notes were magenta before either balance was tried.
//
// Blocks and not samples: a block is a few thousand sensor pixels and its mean is
// what the tone curve is fitted on, so the two read the same frame. A clipped block
// is skipped on both sides — the camera's rendering has no ratio left in it and
// neither has the sensor — and a frame that offers too few blocks for a balance
// keeps the tags above rather than a figure read off noise.
export function previewWhiteBalance(
  data: ArrayLike<number>,
  w: number,
  h: number,
  grid: Uint8Array,
  n: number,
  white: number
): number[] | null {
  const sums = [0, 0, 0];
  let count = 0;
  const clip = white * 0.95;
  for (let by = 0; by < n; by++) {
    for (let bx = 0; bx < n; bx++) {
      const i = (by * n + bx) * 4;
      const r = grid[i];
      const g = grid[i + 1];
      const b = grid[i + 2];
      const hi = Math.max(r, g, b);
      const lo = Math.min(r, g, b);
      if (lo < 160 || hi - lo > 24 || hi > 250) continue;
      const x0 = Math.floor((bx * w) / n);
      const x1 = Math.max(x0 + 1, Math.floor(((bx + 1) * w) / n));
      const y0 = Math.floor((by * h) / n);
      const y1 = Math.max(y0 + 1, Math.floor(((by + 1) * h) / n));
      for (let y = y0; y < y1; y++) {
        let o = (y * w + x0) * 3;
        for (let x = x0; x < x1; x++, o += 3) {
          const pr = data[o];
          const pg = data[o + 1];
          const pb = data[o + 2];
          if (pr > clip || pg > clip || pb > clip) continue;
          sums[0] += pr;
          sums[1] += pg;
          sums[2] += pb;
          count++;
        }
      }
    }
  }
  if (count < 4096 || sums[0] <= 0 || sums[1] <= 0 || sums[2] <= 0) return null;
  const mul = [sums[1] / sums[0], 1, sums[1] / sums[2], 1];
  return mul.every((v) => v > 0.4 && v < 3) ? mul : null;
}

export function getCamToSrgbMatrix(cd: any, brand?: CameraBrand): number[][] {
  if (cd?.rgb_cam && Array.isArray(cd.rgb_cam) && cd.rgb_cam.length >= 3) {
    return [0, 1, 2].map((i) => [0, 1, 2].map((j) => cd.rgb_cam[i]?.[j] || 0));
  }

  if (cd?.cam_xyz && Array.isArray(cd.cam_xyz) && cd.cam_xyz.length >= 3) {
    const cx = cd.cam_xyz;
    const scale = (Math.abs(cx[0][0]) > 100) ? 10000.0 : 1.0;
    const m = [0, 1, 2].map((i) =>
      [0, 1, 2].map((j) =>
        XYZ_TO_SRGB[i][0] * ((cx[j]?.[0] || 0) / scale) +
        XYZ_TO_SRGB[i][1] * ((cx[j]?.[1] || 0) / scale) +
        XYZ_TO_SRGB[i][2] * ((cx[j]?.[2] || 0) / scale)
      )
    );
    return m.map((row) => {
      const num = row[0] + row[1] + row[2];
      return Math.abs(num) < 1e-6 ? row : row.map((v) => v / num);
    });
  }

  // Brand-calibrated default matrix when rgb_cam and cam_xyz are absent
  switch (brand) {
    case 'fujifilm':
      return [[1.15, -0.08, -0.07], [-0.15, 1.25, -0.10], [-0.05, -0.20, 1.25]];
    case 'sony':
      return [[1.18, -0.12, -0.06], [-0.12, 1.22, -0.10], [-0.04, -0.18, 1.22]];
    case 'canon':
      return [[1.12, -0.06, -0.06], [-0.10, 1.20, -0.10], [-0.04, -0.16, 1.20]];
    case 'nikon':
      return [[1.14, -0.08, -0.06], [-0.11, 1.21, -0.10], [-0.04, -0.17, 1.21]];
    default:
      return [[1.0, 0, 0], [0, 1.0, 0], [0, 0, 1.0]];
  }
}

// The size a JPEG declares for itself, off its SOF segment — a header read, no
// decoder. Null when the bytes carry no readable frame header.
export function jpegFrameSize(jpeg: Uint8Array): { width: number; height: number } | null {
  for (let i = 2; i + 9 < jpeg.length; ) {
    if (jpeg[i] !== 0xff) {
      i++;
      continue;
    }
    const marker = jpeg[i + 1];
    if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) {
      return { height: (jpeg[i + 5] << 8) | jpeg[i + 6], width: (jpeg[i + 7] << 8) | jpeg[i + 8] };
    }
    i += 2 + ((jpeg[i + 2] << 8) | jpeg[i + 3]);
  }
  return null;
}

// Is the embedded preview the frame itself? Bodies that write one at the frame
// size (Nikon NEF 6016x4016 out of 6032x4032, Sony ARW, an iPhone ProRAW DNG)
// differ from the sensor by a margin row or two; a thumbnail out of the same
// sensor is 1920x1280, a third of the long side, so 90% divides them cleanly.
// Unreadable sizes keep the old behaviour and trust the preview.
function previewIsFrame(preview: Uint8Array, meta: any): boolean {
  const size = jpegFrameSize(preview);
  const frame = Math.max(
    Number(meta?.iwidth) || 0,
    Number(meta?.iheight) || 0,
    Number(meta?.width) || 0,
    Number(meta?.height) || 0,
    Number(meta?.raw_width) || 0,
    Number(meta?.raw_height) || 0
  );
  if (!size || !frame) return false;
  return Math.max(size.width, size.height) >= frame * 0.9;
}

export async function developRaw(bytes: Uint8Array, fileName?: string): Promise<Uint8Array> {
  // The embedded JPEG is the camera's own rendering — exact colour, tone, white
  // balance and picture profile, generated by the body itself. It is the frame
  // only when it is the size of the frame; on most bodies it is a thumbnail
  // (1920x1280 out of a 24MP sensor, measured on a Fuji RAF and a Lumix RW2), so
  // handing it back is what opened a 24MP file as a 1920 wide one. Whether it is
  // the frame is known from the sensor size, which costs one LibRaw open (91ms
  // measured, metadata only — no demosaic) against seconds for the develop.
  const preview = extractEmbeddedJpeg(bytes);

  const raw = new LibRaw();
  try {
    await raw.open(bytes.slice() as unknown as BufferSource, SETTINGS);
    const meta = await raw.metadata(true);
    if (preview && preview.length > 10000 && previewIsFrame(preview, meta)) {
      return preview;
    }
    const img = await raw.imageData();
    const cd = meta?.color_data;
    if (!img || (!cd?.cam_mul && !cd?.pre_mul) || (!cd.rgb_cam && !cd.cam_xyz)) {
      const camPreview = await cameraPreview(raw);
      if (camPreview && camPreview.length > 10000) return camPreview;
      throw new Error('RAW has no colour data');
    }

    const { width: w, height: h } = img;
    const data = img.data as Uint16Array;
    if (!w || !h) throw new Error('RAW decoded to nothing');

    const effect = Skia.RuntimeEffect.Make(RAW_DEVELOP_SKSL);
    if (!effect) throw new Error('develop shader failed to compile');

    const brand = detectCameraBrand(fileName, meta);
    const white = sensorWhite(data, cd.maximum, cd.black);
    // Read before the develop, not after: the balance and the curve are both read
    // off the file's own preview, and the preview map costs one decode either way.
    const refGrid = preview ? previewGrid(preview, w, h, MATCH_GRID) : null;
    const rawMul = (refGrid && previewWhiteBalance(data, w, h, refGrid, MATCH_GRID, white))
      ?? getBrandWbMultipliers(brand, cd);
    const green = rawMul[1] || rawMul[3] || 1;
    const mul = rawMul.map((v: number) => v / green);
    const [r0, r1, r2] = getCamToSrgbMatrix(cd, brand);

    const bandH = Math.max(1, Math.min(h, Math.floor(BAND_PIXELS / w)));
    const f32 = new Float32Array(w * bandH * 4);
    const half = new Uint16Array(w * bandH * 4);
    // gain, mul, m0, m1, m2, crop (y offset of the band), then where the residual
    // has to be read for the frame pixel this one draws.
    const uniforms = new Float32Array(23);
    uniforms[0] = SAMPLE_MAX / white;
    uniforms.set([mul[0], mul[1], mul[2], 0, r0[0], r0[1], r0[2], 0, r1[0], r1[1], r1[2], 0, r2[0], r2[1], r2[2], 0], 1);

    const develop = (residual: PreviewResidual | null) => {
      const surface = Skia.Surface.MakeOffscreen(w, h) ?? Skia.Surface.Make(w, h);
      if (!surface) return null;
      uniforms[21] = residual ? residual.sx : 0;
      uniforms[22] = residual ? residual.sy : 0;
      const resChild = (residual?.img ?? noResidual()).makeShaderOptions(
        Skia.TileMode.Clamp,
        Skia.TileMode.Clamp,
        Skia.FilterMode.Linear,
        Skia.MipmapMode.None
      );
      for (let y0 = 0; y0 < h; y0 += bandH) {
        const rows = Math.min(bandH, h - y0);
        let o = 0;
        for (let i = y0 * w * 3, end = (y0 + rows) * w * 3; i < end; i += 3) {
          f32[o++] = data[i] / SAMPLE_MAX;
          f32[o++] = data[i + 1] / SAMPLE_MAX;
          f32[o++] = data[i + 2] / SAMPLE_MAX;
          f32[o++] = 1;
        }
        f32ToF16(f32, half, w * rows * 4);
        const band = Skia.Image.MakeImage(
          { width: w, height: rows, colorType: Skia.ColorType.RGBA_F16, alphaType: Skia.AlphaType.Unpremul },
          new Uint8Array(half.buffer, 0, w * rows * 8),
          w * 8
        );
        if (!band) throw new Error('band image failed');
        const child = band.makeShaderOptions(
          Skia.TileMode.Clamp,
          Skia.TileMode.Clamp,
          Skia.FilterMode.Nearest,
          Skia.MipmapMode.None
        );
        uniforms[17] = y0;
        const shader = effect.makeShaderWithChildren(uniforms, [child, resChild]);
        const paint = Skia.Paint();
        paint.setShader(shader);
        surface.getCanvas().drawRect(Skia.XYWHRect(0, y0, w, rows), paint);
        surface.flush();
        paint.delete();
        shader.delete();
        child.delete();
        band.delete();
      }
      resChild.delete();

      const shot = surface.makeImageSnapshot();
      surface.dispose();
      return shot;
    };

    const first = develop(null);
    if (!first) throw new Error('no surface for the develop');

    let finalShot = first;
    let refJpeg = preview;
    if (!refJpeg) {
      refJpeg = await cameraPreview(raw).catch(() => null);
    }

    // The file's own rendering, carried onto this develop: what the camera wrote
    // into the frame minus what the develop made of the same frame at that
    // resolution, added back at the sensor's own. Nothing to fit and nothing to
    // table — a body is matched against its own preview the moment it opens.
    const residual = refJpeg ? previewResidual(first, refJpeg, w, h) : null;
    if (residual) {
      const second = develop(residual);
      residual.img.delete();
      if (second) {
        finalShot = second;
        first.dispose();
      }
    }

    const jpeg = finalShot.encodeToBytes(Skia.ImageFormat.JPEG, 92);
    if (finalShot !== first) {
      finalShot.dispose();
    } else {
      first.dispose();
    }

    if (!jpeg?.length) throw new Error('develop produced no bytes');
    return jpeg;
  } catch (err) {
    // The preview is also the fallback, as it always was: a file whose sensor
    // will not decode still opens as the camera rendered it.
    if (preview && preview.length > 10000) return preview;
    const fallback = await rawThumbnail(bytes).catch(() => null);
    if (fallback) return fallback;
    throw err;
  } finally {
    try {
      raw.dispose();
    } catch {}
  }
}