// The colour the camera itself wrote into the file. // // LibRaw hands back the sensor's own data and nothing about how the body chose // to render it: the white balance it locked, its picture style, its tone curve. // The file carries the answer anyway — the preview saved inside it, which is the // frame the photographer saw and the one a desktop viewer shows. So the develop // is matched to that preview: develop a grid of the frame, take the same grid out // of the embedded preview, and least-squares the 3x3 that takes the first to the // second — then, on what that 3x3 leaves, the tone curve a matrix cannot hold. // // Measured on the A5100 frame the studio was reported on (dE00 against the // camera's own 24MP JPEG, blocks of a 64x64 grid): 12.0 as the develop left it, // 9.2 with this fit drawn in linear light, 6.4 painted through the 8-bit colour // filter this used to be, 6.6 with the 3x3 alone in float, 4.4 with the tone // curve under it. What is left either way is the camera's own local rendering — // its sharpening, the detail a 1.7MP preview never had — and no per-pixel // transform reaches that. // // The fit is on the sRGB-encoded values, exposure included — matrix and curve // alike: the reference is the file's own frame, so matching its brightness is // part of matching its colour. Fitted in that encoded space, not in linear light, // because a least squares in linear light weighs the highlights and comes back // with the midtones wrong — the same frames scored dE00 6.4 fitted encoded // against 9.2 fitted linear. Any body, any picture style, and no fitted table to // go stale — a preview is the one profile every RAW already carries. // Row-major 3x3, on the sRGB-encoded develop: what the shader applies to a pixel. export type Mat3 = readonly [ number, number, number, number, number, number, number, number, number, ]; // The grid the fit reads, and the size of the preview map handed to it. export const MATCH_GRID = 64; // A block at or past this has nothing left to say about a rendering. const CLIP = 0.97; const FLOOR = 0.03; // The per-file colour fit: a 3x3, then the tone curve a 3x3 cannot hold — one // cubic a channel, on the encoded value the 3x3 leaves. `tone` is that curve in // the shader's own order: r a b c d, then g, then b. export interface Match { m: Mat3; tone: Float32Array; } // What the second stage does when the frame has no curve the fit can trust. export const FLAT_TONE = new Float32Array([0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0]); // Gaussian elimination on the one system this file fits: a channel is a row of // three, for the 3x3 and again for the 3x3's own error. function solve(C: number[][], d: number[]): number[] { const n = 3; const a = Array.from({ length: n }, (_, i) => [...C[i], d[i]]); for (const row of a) for (let c = 0; c < n; c++) row[c] += 1e-9; for (let c = 0; c < n; c++) { let p = c; for (let r = c + 1; r < n; r++) if (Math.abs(a[r][c]) > Math.abs(a[p][c])) p = r; [a[c], a[p]] = [a[p], a[c]]; for (let r = 0; r < n; r++) { if (r === c) continue; const f = a[r][c] / a[c][c]; for (let k = c; k <= n; k++) a[r][k] -= f * a[c][k]; } } return a.map((_, i) => a[i][n] / a[i][i]); } // The curve a channel needs to reach the camera's, least squared: what the 3x3 // left for that channel, a cubic against the block the camera wrote there. // // The cubic is pinned at BOTH black (x=0 -> y=0) and white (x=1 -> y=1). // Pinned at both ends, a blown pixel (1,1,1) stays pure white, and a deep shadow // pixel (0,0,0) stays pure black without shifting to purple or magenta noise. function basis(i: number, x: number): number { const x3 = x * x * x; return i === 0 ? x - x3 : x * x - x3; } function solve2x2(A: number[][], d: number[]): number[] { const det = A[0][0] * A[1][1] - A[0][1] * A[1][0] + 1e-9; const a = (d[0] * A[1][1] - d[1] * A[0][1]) / det; const b = (A[0][0] * d[1] - A[1][0] * d[0]) / det; return [a, b]; } function fitTone(blocks: { t: number[]; q: number[] }[]): Float32Array { const out = new Float32Array(12); for (let c = 0; c < 3; c++) { const A = [[0, 0], [0, 0]]; const d = [0, 0]; for (const { t, q } of blocks) { const x = q[c], b = [basis(0, x), basis(1, x)]; for (let p = 0; p < 2; p++) { for (let k = 0; k < 2; k++) A[p][k] += b[p] * b[k]; d[p] += b[p] * (t[c] - x * x * x); } } const [a, b] = solve2x2(A, d); out.set([0, a, b, 1 - a - b], c * 4); } return out; } // A curve that comes back folded, or wandered far from the value it was handed, // is a fit off blocks that fought each other, not a rendering: the caller keeps // the 3x3 alone rather than a look nobody's camera has. The floor sits below the // zero a curve is allowed to reach — the develop clamps there anyway — and the // slope is only asked not to turn back. function curveIsSane(tone: Float32Array): boolean { for (const v of tone) if (!Number.isFinite(v) || Math.abs(v) > 4) return false; for (let c = 0; c < 3; c++) { const [a, b, cc, dd] = [tone[c * 4], tone[c * 4 + 1], tone[c * 4 + 2], tone[c * 4 + 3]]; let prev = a; for (let i = 1; i <= 16; i++) { const x = i / 16; const y = a + x * (b + x * (cc + x * dd)); if (y < prev - 0.12 || y < -0.15 || y > 1.15) return false; prev = y; } } return true; } // `dev` and `ref` are n x n RGBA maps of the same frame, one byte per channel. // The 3x3 comes back on those encoded values — the space the caller draws it in. // Null when the two cannot be related — too few blocks carry colour, or the // solution runs away — and the caller develops the frame as the sensor left it. export function fitMatch(dev: Uint8Array, ref: Uint8Array, n = MATCH_GRID): Match | null { const C = [[0, 0, 0], [0, 0, 0], [0, 0, 0]]; const d = [[0, 0, 0], [0, 0, 0], [0, 0, 0]]; const blocks: { e: number[]; t: number[] }[] = []; let used = 0; for (let i = 0; i < n * n; i++) { const e = [dev[i * 4] / 255, dev[i * 4 + 1] / 255, dev[i * 4 + 2] / 255]; const t = [ref[i * 4] / 255, ref[i * 4 + 1] / 255, ref[i * 4 + 2] / 255]; // A block the camera clipped has no colour left to copy, one at the floor is // only the pedestal, and one the develop clipped has lost the ratio between // its channels — none of them say anything about the rendering. if (t[0] > CLIP && t[1] > CLIP && t[2] > CLIP) continue; if (e[0] > CLIP || e[1] > CLIP || e[2] > CLIP) continue; if (Math.max(t[0], t[1], t[2]) < FLOOR) continue; used++; blocks.push({ e, t }); for (let x = 0; x < 3; x++) { for (let y = 0; y < 3; y++) C[x][y] += e[x] * e[y]; for (let c = 0; c < 3; c++) d[x][c] += e[c] * t[x]; } } // A fit off a handful of blocks is a fit off noise. if (used < (n * n) / 8) return null; // Ridge towards identity, scaled to the data. The three channels of a develop // are near-collinear — a camera's own rendering of a neutral scene is close to // neutral — so the plain normal equations come back singular-ish, and the frame's // own noise decides a matrix whose diagonal lands at or below zero. λ of 1e-3 of // the data's scale lifts that back to a definite system and moves the fitted // matrix only in the third decimal (measured: 0.001 kept the tone the same to // 0.3% where 0.1 began flattening it). const lambda = 1e-3 * ((C[0][0] + C[1][1] + C[2][2]) / 3); for (let r = 0; r < 3; r++) { C[r][r] += lambda; d[r][r] += lambda; } const m = [...solve(C, d[0]), ...solve(C, d[1]), ...solve(C, d[2])]; // A row that ran off is a degenerate fit, not a look. A diagonal element at or // below zero is not one: what these matrices hold is the difference between two // near-identical channels (X-T3 -0.04, X100V -1.67 on a row that carries +1.05 // and -1.67), and refusing those refused the whole fit — those bodies opened // byte-identical to an unfitted develop. for (const v of m) if (!Number.isFinite(v) || Math.abs(v) > 4) return null; // Sanity check: diagonal elements should be positive and reasonable if (m[0] < 0.1 || m[4] < 0.1 || m[8] < 0.1 || m[0] > 3.0 || m[4] > 3.0 || m[8] > 3.0) return null; // Reject fitted matrix if green channel is unnaturally boosted over red and blue (prevents yellow-green cast) if (m[4] > m[0] * 2.5 || m[4] > m[8] * 2.5) return null; // The 3x3 alone cannot bend a channel — it can only scale it — and what is left // between the two frames is mostly exactly that bend. So the residual after the // 3x3 is fitted a cubic a channel, on the value the 3x3 leaves. const q = blocks.map(({ e, t }) => { const v = [ m[0] * e[0] + m[1] * e[1] + m[2] * e[2], m[3] * e[0] + m[4] * e[1] + m[5] * e[2], m[6] * e[0] + m[7] * e[1] + m[8] * e[2], ]; return { t, q: v.map((x) => Math.min(1, Math.max(0, x))) }; }); const tone = fitTone(q); return { m: m as unknown as Mat3, tone: curveIsSane(tone) ? tone : FLAT_TONE }; }