diff --git a/docker/frontend/scripts/raw-develop-check.mjs b/docker/frontend/scripts/raw-develop-check.mjs index 9d036d5..185a5f4 100644 --- a/docker/frontend/scripts/raw-develop-check.mjs +++ b/docker/frontend/scripts/raw-develop-check.mjs @@ -36,9 +36,12 @@ assert.match(code, /float3 lin = n \* mul\.rgb;/); // across the whole sky. Measured on that frame in the app against the camera's own // JPEG: mean dRGB +1.2, -5.9, -6.1 with the sensor's clip alone, +0.8, +1.0, +1.1 // with both, mean |dL| 21.5 against 11.1. +// The gate opens below the white level, not at it: 0.82 against 0.98 was measured +// after the pairs above, so the tint reads on the way up rather than only once the +// frame has already run out of range. assert.match( code, - /float hi = max\(max\(n\.r, n\.g\), n\.b\);\s*hi = max\(hi, max\(max\(lin\.r, lin\.g\), lin\.b\)\);\s*float blown = smoothstep\(0\.99, 1\.02, hi\);\s*rgb = mix\(rgb, float3\(mx\), blown\);/ + /float hi = max\(max\(n\.r, n\.g\), n\.b\);\s*hi = max\(hi, max\(max\(lin\.r, lin\.g\), lin\.b\)\);\s*float blown = smoothstep\(0\.82, 0\.98, hi\);\s*rgb = mix\(rgb, float3\(mx\), blown\);/ ); // The inverse is built on the CPU side and has to be the first uniform of the @@ -46,7 +49,8 @@ assert.match( // the frame itself ran out at — `maximum` alone left every frame a stop bright, // and a fixed factor two only fitted the two bodies it was measured on. The // probe and its own check live in src/engine/sensorWhite.ts. -assert.match(src, /^ *uniforms\[0\] = SAMPLE_MAX \/ sensorWhite\(data, cd\.maximum, cd\.black\);$/m); +assert.match(src, /^ *const white = sensorWhite\(data, cd\.maximum, cd\.black\);$/m); +assert.match(src, /^ *uniforms\[0\] = SAMPLE_MAX \/ white;$/m); assert.match(src, /^import \{ sensorWhite \} from '\.\/sensorWhite';$/m); // On the FX30 that inverse is 2.065 (65535 / (2 (16380 - 512))), against the @@ -74,9 +78,33 @@ assert.doesNotMatch(code, /uniform float4 w0;/, 'the body table is back in the s assert.doesNotMatch(src, /fitMatch/, 'the 3x3 preview match is back'); assert.doesNotMatch(code, /uniform float4 f0;/, 'the 3x3 preview match is back in the shader'); assert.match(src, /const first = develop\(null\);/); -assert.match(src, /const jpeg = first\.encodeToBytes/); +assert.match(src, /const jpeg = finalShot\.encodeToBytes\(Skia\.ImageFormat\.JPEG, 92\);/); assert.match(src, /preview && preview\.length > 10000/); +// The balance the frame opens with is read off the file's own preview — the same +// reference the tone curve is fitted to — and only falls back to LibRaw's tags when +// the preview has nothing neutral and bright left to read. The tags alone were not +// the balance: on the Fujifilm RAF this was reported on, `pre_mul` was short of blue +// twice over (R/G 2.296 against the frame's 1.795, B/G 1.307 against 1.626) and left +// every white pink, `cam_mul` came closer, and the pair read off the preview matched +// it: block a* -1.9 against the preview's -2.1 where the tags gave 0.0 and -1.0, +// dE00 4.93 where they gave 6.73 and 5.27. And it holds across bodies for the same +// reason — it is read on the frame, not on the tags. +assert.ok( + src.indexOf('return cd.cam_mul;') < src.indexOf('return cd.pre_mul;'), + 'pre_mul is preferred over the frame\'s own balance again' +); +assert.match(src, /^ *const refGrid = preview \? previewGrid\(preview, w, h, MATCH_GRID\) : null;$/m); +assert.match( + src, + /const rawMul = \(refGrid && previewWhiteBalance\(data, w, h, refGrid, MATCH_GRID, white\)\)\s*\?\? getBrandWbMultipliers\(brand, cd\);/ +); +assert.match(src, /^export function previewWhiteBalance\($/m); +// ...and a balance read off too little of the frame is no balance: a frame that +// offers fewer than this many sensor pixels across its neutral bright blocks keeps +// the tags rather than a ratio read off noise. +assert.match(src, /if \(count < 4096 \|\| sums\[0\] <= 0 \|\| sums\[1\] <= 0 \|\| sums\[2\] <= 0\) return null;/); + // The fit has to land on the encoded value in float, not as an 8-bit colour filter // painted over the frame afterwards: the fit carries an exposure, and a channel it // lifted past the white level was cut where it stood — blue first, the channel the @@ -127,11 +155,16 @@ assert.match(src, /^ *halfSize: false,$/m, 'the develop is half-size again'); // the cast the frame opened with. And the matrix must not be divided by `pre_mul` // again: that undoes the row-sum-to-one normalisation dcraw built into rgb_cam. { - const body = src.match(/const XYZ_TO_SRGB = \[[\s\S]*?\n\}\n/)?.[0]; - assert.ok(body, 'getCamToSrgbMatrix not found'); - // The file is TypeScript; the two annotations in this block are all that stop + const xyz = src.match(/^const XYZ_TO_SRGB = \[[\s\S]*?\n\];$/m)?.[0]; + const fn = src.match(/^export function getCamToSrgbMatrix\([\s\S]*?\n\}\n/m)?.[0]; + assert.ok(xyz && fn, 'getCamToSrgbMatrix not found'); + // The file is TypeScript; the annotations in this block are all that stop // `new Function` from reading it. - const js = body.replace(/: any/g, '').replace(/: number\[\]\[\]/g, ''); + const js = (xyz + '\n' + fn) + .replace(/^export /m, '') + .replace(/: any/g, '') + .replace(/: number\[\]\[\]/g, '') + .replace(/\?: CameraBrand/g, ''); const getCamToSrgbMatrix = new Function(`${js}; return getCamToSrgbMatrix;`)(); const rgb_cam = [ [1.631760597229004, -0.39896339178085327, -0.23279713094234467], @@ -163,4 +196,66 @@ assert.match(src, /^ *halfSize: false,$/m, 'the develop is half-size again'); } } +// The balance read off the preview is arithmetic on the plane and the preview +// map, so both can be handed to it here. A plane whose red stands 1.5x its green +// and whose blue 1.2x, under a map the camera left bright and neutral, has one +// balance that would equalise those blocks, and it is the one that comes back. +{ + const fn = src.match(/^export function previewWhiteBalance\([\s\S]*?\n\}\n/m)?.[0]; + assert.ok(fn, 'previewWhiteBalance not found'); + const previewWhiteBalance = new Function( + `${fn.replace(/^export /m, '').replace(/:\s*(ArrayLike|Uint8Array|number\[\] \| null|number)/g, '')}; return previewWhiteBalance;` + )(); + + const N = 4; + const W = 128; + const H = 128; + const WHITE = 10000; + const plane = (r, g, b) => { + const d = new Uint16Array(W * H * 3); + for (let i = 0; i < W * H; i++) { + d[i * 3] = r; + d[i * 3 + 1] = g; + d[i * 3 + 2] = b; + } + return d; + }; + const block = (g, bx, by, r, gg, b) => { + const i = (by * N + bx) * 4; + g[i] = r; + g[i + 1] = gg; + g[i + 2] = b; + g[i + 3] = 255; + }; + const map = (f) => { + const g = new Uint8Array(N * N * 4); + for (let by = 0; by < N; by++) for (let bx = 0; bx < N; bx++) f(g, bx, by); + return g; + }; + const allWhite = map((g, bx, by) => block(g, bx, by, 200, 200, 200)); + + const mul = previewWhiteBalance(plane(3000, 2000, 2400), W, H, allWhite, N, WHITE); + assert.ok(mul, 'a frame of bright neutral blocks has to read a balance'); + assert.ok( + Math.abs(mul[0] - 2 / 3) < 1e-6 && mul[1] === 1 && Math.abs(mul[2] - 5 / 6) < 1e-6, + `read ${mul} off a 1.5x red / 1.2x blue plane` + ); + + // A block the camera did not leave neutral is not read, however bright it is: + // it carries the scene's colour, not the frame's balance. + const tinted = previewWhiteBalance( + plane(3000, 2000, 2400), + W, + H, + map((g, bx, by) => (bx ? block(g, bx, by, 200, 200, 200) : block(g, bx, by, 200, 120, 200))), + N, + WHITE + ); + assert.deepEqual(tinted, mul, 'a tinted block moved the balance'); + + // Too little of the frame left to read is no balance at all — the caller keeps + // the tags rather than a ratio read off a handful of pixels. + assert.equal(previewWhiteBalance(plane(3000, 2000, 2400), 30, 30, allWhite, N, WHITE), null); +} + console.log('raw-develop-check ok'); diff --git a/docker/frontend/src/engine/rawDevelop.ts b/docker/frontend/src/engine/rawDevelop.ts index 37bf81e..6d4b46a 100644 --- a/docker/frontend/src/engine/rawDevelop.ts +++ b/docker/frontend/src/engine/rawDevelop.ts @@ -558,17 +558,21 @@ export function detectCameraBrand(fileName?: string, meta?: any): CameraBrand { } export function getBrandWbMultipliers(brand: CameraBrand, cd: any): number[] { - // 1. Try pre-calculated WB multipliers from LibRaw (pre_mul) - // LibRaw pre_mul contains green-normalized, daylight-calibrated WB multipliers. - if (cd?.pre_mul && Array.isArray(cd.pre_mul) && cd.pre_mul.some((v: number) => v > 0 && v !== 1)) { - return cd.pre_mul; - } - - // 2. Try camera WB multipliers from LibRaw (cam_mul) + // 1. The balance the body recorded with the frame. `pre_mul` is the daylight one + // LibRaw carries for the model, and it is what used to be read first — on the + // Fujifilm RAF this was reported on it is the wrong one twice over (R/G 2.296 + // against the frame's own 1.795, B/G 1.307 against 1.626), which left blue short + // and every white in the frame pink. Measured against the file's own preview over + // the sample set, dE00 6.73 -> 5.27 on that frame and 7.52 -> 7.26 on the X-T20's. if (cd?.cam_mul && Array.isArray(cd.cam_mul) && cd.cam_mul.some((v: number) => v > 0 && v !== 1)) { return cd.cam_mul; } + // 2. Otherwise the daylight balance LibRaw worked out for the model. + if (cd?.pre_mul && Array.isArray(cd.pre_mul) && cd.pre_mul.some((v: number) => v > 0 && v !== 1)) { + return cd.pre_mul; + } + // 3. Brand-calibrated WB fallbacks (prevents generic single fallback from producing color casts) switch (brand) { case 'fujifilm': @@ -595,7 +599,73 @@ export function getBrandWbMultipliers(brand: CameraBrand, cd: any): number[] { } } -function getCamToSrgbMatrix(cd: any, brand?: CameraBrand): number[][] { +// The balance the file's own preview was rendered with. +// +// Neither balance LibRaw hands over is it. `cam_mul` is what the body recorded and +// `pre_mul` the daylight one it carries for the model, and the preview in the file +// is the rendering the body actually made — so the balance is in there, read off the +// blocks the camera left bright and near neutral: average the sensor triple behind +// each of those blocks over the block's own pixels, and the triple that makes those +// blocks come out equal is the frame's white. The camera's rendering is the same +// reference the tone curve below is fitted to; this is its WB row. +// +// Read on the frame, not on the file's tags, which is why it holds across bodies: +// measured against the embedded preview, the Fujifilm RAF this was reported on came +// back at block a* -1.9 against the preview's -2.1 where `pre_mul` gave 0.0 and +// `cam_mul` -1.0 — the pink in every white of the frame — and dE00 4.93 where they +// gave 6.73 and 5.27. On the X-T20 frame: -4.2 against -3.9, dE00 6.56 against 7.52 +// and 7.26. Both notes were magenta before either balance was tried. +// +// Blocks and not samples: a block is a few thousand sensor pixels and its mean is +// what the tone curve is fitted on, so the two read the same frame. A clipped block +// is skipped on both sides — the camera's rendering has no ratio left in it and +// neither has the sensor — and a frame that offers too few blocks for a balance +// keeps the tags above rather than a figure read off noise. +export function previewWhiteBalance( + data: ArrayLike, + w: number, + h: number, + grid: Uint8Array, + n: number, + white: number +): number[] | null { + const sums = [0, 0, 0]; + let count = 0; + const clip = white * 0.95; + for (let by = 0; by < n; by++) { + for (let bx = 0; bx < n; bx++) { + const i = (by * n + bx) * 4; + const r = grid[i]; + const g = grid[i + 1]; + const b = grid[i + 2]; + const hi = Math.max(r, g, b); + const lo = Math.min(r, g, b); + if (lo < 160 || hi - lo > 24 || hi > 250) continue; + const x0 = Math.floor((bx * w) / n); + const x1 = Math.max(x0 + 1, Math.floor(((bx + 1) * w) / n)); + const y0 = Math.floor((by * h) / n); + const y1 = Math.max(y0 + 1, Math.floor(((by + 1) * h) / n)); + for (let y = y0; y < y1; y++) { + let o = (y * w + x0) * 3; + for (let x = x0; x < x1; x++, o += 3) { + const pr = data[o]; + const pg = data[o + 1]; + const pb = data[o + 2]; + if (pr > clip || pg > clip || pb > clip) continue; + sums[0] += pr; + sums[1] += pg; + sums[2] += pb; + count++; + } + } + } + } + if (count < 4096 || sums[0] <= 0 || sums[1] <= 0 || sums[2] <= 0) return null; + const mul = [sums[1] / sums[0], 1, sums[1] / sums[2], 1]; + return mul.every((v) => v > 0.4 && v < 3) ? mul : null; +} + +export function getCamToSrgbMatrix(cd: any, brand?: CameraBrand): number[][] { if (cd?.rgb_cam && Array.isArray(cd.rgb_cam) && cd.rgb_cam.length >= 3) { return [0, 1, 2].map((i) => [0, 1, 2].map((j) => cd.rgb_cam[i]?.[j] || 0)); } @@ -700,7 +770,12 @@ export async function developRaw(bytes: Uint8Array, fileName?: string): Promise< if (!effect) throw new Error('develop shader failed to compile'); const brand = detectCameraBrand(fileName, meta); - const rawMul = getBrandWbMultipliers(brand, cd); + const white = sensorWhite(data, cd.maximum, cd.black); + // Read before the develop, not after: the balance and the curve are both read + // off the file's own preview, and the preview map costs one decode either way. + const refGrid = preview ? previewGrid(preview, w, h, MATCH_GRID) : null; + const rawMul = (refGrid && previewWhiteBalance(data, w, h, refGrid, MATCH_GRID, white)) + ?? getBrandWbMultipliers(brand, cd); const green = rawMul[1] || rawMul[3] || 1; const mul = rawMul.map((v: number) => v / green); const [r0, r1, r2] = getCamToSrgbMatrix(cd, brand); @@ -709,7 +784,7 @@ export async function developRaw(bytes: Uint8Array, fileName?: string): Promise< const f32 = new Float32Array(w * bandH * 4); const half = new Uint16Array(w * bandH * 4); const uniforms = new Float32Array(33); - uniforms[0] = SAMPLE_MAX / sensorWhite(data, cd.maximum, cd.black); + uniforms[0] = SAMPLE_MAX / white; uniforms.set([mul[0], mul[1], mul[2], 0, r0[0], r0[1], r0[2], 0, r1[0], r1[1], r1[2], 0, r2[0], r2[1], r2[2], 0], 1); const develop = (tone: Float32Array | null) => { @@ -767,9 +842,9 @@ export async function developRaw(bytes: Uint8Array, fileName?: string): Promise< if (refJpeg) { try { const devGrid = gridOf(first, MATCH_GRID); - const refGrid = previewGrid(refJpeg, w, h, MATCH_GRID); - if (devGrid && refGrid) { - const fittedTone = toneMatch(devGrid, refGrid, MATCH_GRID); + const matchGrid = refGrid ?? previewGrid(refJpeg, w, h, MATCH_GRID); + if (devGrid && matchGrid) { + const fittedTone = toneMatch(devGrid, matchGrid, MATCH_GRID); if (fittedTone) { const second = develop(fittedTone); if (second) {