diff --git a/docker/frontend/scripts/highlight-knee-check.mjs b/docker/frontend/scripts/highlight-knee-check.mjs new file mode 100644 index 0000000..e07fe3c --- /dev/null +++ b/docker/frontend/scripts/highlight-knee-check.mjs @@ -0,0 +1,86 @@ +// Highlight roll-off, both ends of the pipeline, as one soft knee: +// +// L' = L , L < T +// L' = T + (L - T) / (1 + 2 S (L - T)) , L >= T +// +// The develop draws it on the sensor's own levels (T = 0.7, S = 1 / (2 (1 - T)), +// which puts the asymptote on 1.0) so the two stops the sensor holds above its +// white level are COMPRESSED into the frame instead of being thrown away by the +// old fade-to-white; the tone pass draws the same curve in linear light on the +// value the develop and the camera match left, where -HL is the knob (T = 0.5, +// S = |hl|). Before this, a blown sky left the develop on exactly 1.0 in all +// three channels and HL had a flat white to pull on: measured on DSC03453.ARW, +// where the camera's own preview is clipped the develop's luma was 253.4 with a +// standard deviation of 2.4, against 251.2 / 10.0 through the knee. +// +// Both are SkSL, so the shape is pinned on the source; the curve itself is +// checked as arithmetic, with the constants the source is asserted to carry. +// +// node scripts/highlight-knee-check.mjs +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; + +const develop = readFileSync(new URL('../src/engine/rawDevelop.ts', import.meta.url), 'utf8'); +const tone = readFileSync(new URL('../shared/utils/toneShader.ts', import.meta.url), 'utf8'); + +// The develop: knee on the sensor's max channel, the channel ratios kept, so the +// hue and the saturation of a blown area survive the pull-down. +const dev = develop.match(/if \(mx > 0\.7\) \{[\s\S]*?\n \}/)?.[0]; +assert.ok(dev, 'the develop knee is gone — a blown sky is flat 1.0 again'); +assert.match(dev, /float over = mx - 0\.7;/); +assert.match(dev, /rgb \*= \(0\.7 \+ over \/ \(1\.0 \+ over \* 3\.3333\)\) \/ mx;/); +assert.doesNotMatch(develop, /mix\(rgb \/ mx, float3\(1\.0\)/, 'the fade-to-white is back'); + +// The tone pass: the knee runs in LINEAR light and before the luma is read, the +// lift keeps its headroom weight — and the recovery must not also ride the +// additive term, which would darken the white the knee protects. +assert.match(tone, /if \(hl < 0\.0\) \{/); +assert.match(tone, /vec3 lin = toLinear\(rgb\);/); +assert.match(tone, /float l0 = dot\(lin, vec3\(0\.2126, 0\.7152, 0\.0722\)\);/); +assert.match(tone, /lin \*= \(0\.5 \+ over \/ \(1\.0 \+ S \* over \* 2\.0\)\) \/ l0;/); +assert.match(tone, /rgb = clamp\(toEncoded\(lin\), 0\.0, 1\.0\);/); +assert.match(tone, /float o = t \+ max\(hl, 0\.0\) \* hlMask \* \(1\.0 - t\) \+ sh \* 0\.34 \* shMask;/); +// The transfer pair has to be the accurate one, or the knee is drawn in a space +// that is not linear at all. +assert.match(tone, /return mix\(c \/ 12\.92, pow\(\(c \+ 0\.055\) \/ 1\.055, vec3\(2\.4\)\), step\(vec3\(0\.04045\), c\)\);/); + +// The arithmetic. T = 0.7 / S = 1 / (2 (1 - T)) is the develop's pair (S is what +// puts the asymptote on 1.0: T + 1/(2S) = 1); T = 0.5 with S = 1 is the top of +// the tone pass's knob. +const knee = (l, T, S) => (l < T ? l : T + (l - T) / (1 + 2 * S * (l - T))); + +for (const [T, S] of [ + [0.7, 1 / (2 * (1 - 0.7))], + [0.5, 0.25], + [0.5, 1], +]) { + // Below the knee the frame is untouched, and the curve is continuous and C1 at + // T — slope 1 on both sides — so there is no seam for a later pass to mask. + assert.equal(knee(T - 0.2, T, S), T - 0.2); + assert.equal(knee(T, T, S), T); + const slope = (x) => (knee(x + 1e-6, T, S) - knee(x, T, S)) / 1e-6; + assert.ok(Math.abs(slope(T) - 1) < 1e-3, `seam at T=${T}: slope ${slope(T)}`); + // Monotone, and never a brightening: a recovery slider that lifted a highlight + // would be a lift in disguise, and an inverted pair of pixels is a visible edge. + let prev = -Infinity; + for (let l = 0; l <= 2; l += 1 / 512) { + assert.ok(slope(l) > 0, `inverted at ${l} (T=${T}, S=${S})`); + assert.ok(knee(l, T, S) <= l + 1e-9, `brightened ${l} -> ${knee(l, T, S)}`); + assert.ok(knee(l, T, S) >= prev); + prev = knee(l, T, S); + } + // The asymptote: everything the sensor held above the knee lands under it. + // The develop's pair puts it exactly on 1.0, the tone pass's S = 1 on 0.75. + assert.ok(Math.abs(knee(1e6, T, S) - (T + 1 / (2 * S))) < 1e-4); +} +assert.ok(Math.abs(0.7 + 1 / (2 * (1 / (2 * (1 - 0.7)))) - 1) < 1e-9, 'the develop plateau left 1.0'); +// ...and the same pair in the encoded domain, which is the domain the develop +// hands over: mx = 1.0 (the white level) lands on 237, the sensor's own plateau +// (1.93, see the gain above) on 248 — a ramp of a dozen code values where the +// old fade-to-white left nothing above 250 at all. +const enc = (x) => (x <= 0.0031308 ? x * 12.92 : 1.055 * x ** (1 / 2.4) - 0.055); +const DEV_S = 1 / (2 * (1 - 0.7)); +assert.equal(Math.round(enc(knee(1.0, 0.7, DEV_S)) * 255), 237); +assert.equal(Math.round(enc(knee(1.93, 0.7, DEV_S)) * 255), 248); + +console.log('highlight-knee-check ok'); diff --git a/docker/frontend/scripts/preview-match-check.mjs b/docker/frontend/scripts/preview-match-check.mjs new file mode 100644 index 0000000..a3ae1d3 --- /dev/null +++ b/docker/frontend/scripts/preview-match-check.mjs @@ -0,0 +1,102 @@ +// The per-file colour fit is pure arithmetic, so it can be checked here rather +// than in a browser: hand fitMatch a grid and the same grid through a known 3x3, +// and it has to hand that 3x3 back. Done that way because the function only ever +// runs inside the RAW develop, which needs LibRaw and CanvasKit — neither of +// which a node check has. The module is TypeScript, so it is transpiled on the +// fly out of the installed compiler (the repo's convention for checks: see +// raw-develop-check.mjs, which reads the shader source instead). +// +// node scripts/preview-match-check.mjs +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import ts from 'typescript'; + +const file = new URL('../src/engine/previewMatch.ts', import.meta.url); +const js = ts.transpileModule(readFileSync(file, 'utf8'), { + compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2022 }, +}).outputText; +const { fitMatch, MATCH_GRID } = await import( + `data:text/javascript;base64,${Buffer.from(js).toString('base64')}` +); + +// A grid with colour in it: three independent ramps, none of them clipped and +// none at the floor, so every block carries something to fit. Independent on +// purpose — three correlated channels leave the 3x3 ill-conditioned, and a fit +// that comes back a percent off on such a scene says nothing about the code. +function scene() { + let s = 1; + const rnd = () => ((s = (s * 1103515245 + 12345) & 0x7fffffff) % 4096) / 4096; + const g = new Uint8Array(MATCH_GRID * MATCH_GRID * 4); + for (let i = 0; i < MATCH_GRID * MATCH_GRID; i++) { + for (let c = 0; c < 3; c++) g[i * 4 + c] = 25 + Math.round(rnd() * 190); + g[i * 4 + 3] = 255; + } + return g; +} + +const M = [1.06, -0.05, 0.01, 0.02, 0.98, 0.03, -0.04, 0.06, 1.02]; +const dev = scene(); +const ref = new Uint8Array(dev.length); +for (let i = 0; i < MATCH_GRID * MATCH_GRID; i++) { + const e = [dev[i * 4], dev[i * 4 + 1], dev[i * 4 + 2]]; + for (let r = 0; r < 3; r++) { + const v = M[r * 3] * e[0] + M[r * 3 + 1] * e[1] + M[r * 3 + 2] * e[2]; + ref[i * 4 + r] = Math.max(0, Math.min(255, Math.round(v))); + } + ref[i * 4 + 3] = 255; +} + +const fit = fitMatch(dev, ref); +assert.ok(fit, 'a fitted grid has to fit'); +for (let r = 0; r < 3; r++) { + for (let c = 0; c < 3; c++) { + assert.ok(Math.abs(fit.m[r * 3 + c] - M[r * 3 + c]) < 0.01, `recovered ${fit.m[r * 3 + c]} for ${M[r * 3 + c]}`); + } +} +// This scene was made by a 3x3 and nothing else, so the curve fitted on what the +// 3x3 leaves has to come back flat — the identity, up to the rounding the scene +// carries. +assert.equal(fit.tone.length, 12); +for (let i = 0; i < 12; i++) { + const flat = i % 4 === 1 ? 1 : 0; + assert.ok(Math.abs(fit.tone[i] - flat) < 0.15, `tone ${i} came back ${fit.tone[i]}, not ${flat}`); +} + +// Nothing to fit against: the camera clipped the whole frame, so no block says +// anything about its rendering, and the caller develops as the sensor left it. +assert.equal(fitMatch(dev, new Uint8Array(dev.length).fill(255)), null); + +// A near-neutral frame through a matrix whose rows carry a negative element — what +// the bodies the fit used to refuse looked like (X-T3 -0.04, X100V -1.67 on rows +// that also carry +1.05 and +1.63). Three channels that share nearly all their +// content leave the normal equations near-singular, and the plain solve came back +// with a diagonal element at or below zero, which the caller read as a degeneracy +// and dropped the whole fit for. It is a colour: the ridge lifts the system back +// to definite and the fit has to come back a matrix. +function neutral() { + let s = 7; + const rnd = () => ((s = (s * 1103515245 + 12345) & 0x7fffffff) % 4096) / 4096; + const g = new Uint8Array(MATCH_GRID * MATCH_GRID * 4); + for (let i = 0; i < MATCH_GRID * MATCH_GRID; i++) { + const x = 40 + Math.round(rnd() * 180); + for (let c = 0; c < 3; c++) g[i * 4 + c] = Math.max(0, Math.min(255, x + Math.round((rnd() - 0.5) * 8))); + g[i * 4 + 3] = 255; + } + return g; +} +const N = [1.05, -1.67, 1.63, 0.02, 0.95, 0.03, -0.04, 0.06, 1.02]; +const nDev = neutral(); +const nRef = new Uint8Array(nDev.length); +for (let i = 0; i < MATCH_GRID * MATCH_GRID; i++) { + const e = [nDev[i * 4], nDev[i * 4 + 1], nDev[i * 4 + 2]]; + for (let r = 0; r < 3; r++) { + const v = N[r * 3] * e[0] + N[r * 3 + 1] * e[1] + N[r * 3 + 2] * e[2]; + nRef[i * 4 + r] = Math.max(0, Math.min(255, Math.round(v))); + } + nRef[i * 4 + 3] = 255; +} +const nFit = fitMatch(nDev, nRef); +assert.ok(nFit, 'a near-neutral frame has to fit, not come back null'); +for (const v of nFit.m) assert.ok(Number.isFinite(v) && Math.abs(v) <= 4, `degenerate fit ${v}`); + +console.log('preview-match-check ok'); diff --git a/docker/frontend/scripts/raw-develop-check.mjs b/docker/frontend/scripts/raw-develop-check.mjs new file mode 100644 index 0000000..364da69 --- /dev/null +++ b/docker/frontend/scripts/raw-develop-check.mjs @@ -0,0 +1,122 @@ +// LibRaw subtracts the black level itself, whatever `noAutoScale` says, so the +// develop must not do it again: a second subtraction drained red and blue — the +// channels `cam_mul` lifts most — and turned every Sony ARW green. This pins that +// down on the source, since the develop itself only runs in a browser (LibRaw +// worker + CanvasKit). +// +// node scripts/raw-develop-check.mjs +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; + +const src = readFileSync(new URL('../src/engine/rawDevelop.ts', import.meta.url), 'utf8'); +const sksl = src.match(/const RAW_DEVELOP_SKSL = `([\s\S]*?)`;/)?.[1]; +assert.ok(sksl, 'RAW_DEVELOP_SKSL not found'); +// The prose above the code talks about the black level, so drop the comments. +const code = sksl.replace(/\/\/[^\n]*/g, ''); + +// The plane comes in already black-subtracted, so its only scale is the white +// level less the black level, handed over as one inverse. +assert.match(code, /uniform float gain;/); +assert.doesNotMatch(code, /uniform float4 black;/); +assert.doesNotMatch(code, /-\s*black/, 'the shader subtracts the black level again'); +assert.match(code, /float3 n = max\(p\.rgb \* gain, 0\.0\);/); +assert.match(code, /float3 lin = n \* mul\.rgb;/); + +// The WB gains, applied where the sensor has already clipped, are what leaves the +// blown areas magenta, so the shader has to read the sensor's own levels — before +// the gains — and desaturate the pixel towards its own value as the clip is +// approached. Measured on the FX30 ARW: the camera's own JPEG has an all-white top +// percentile (R/G 0.995, B/G 0.999), the develop without this line came back at +// R/G 1.017, B/G 0.862 — a warm tint on every blown area. +// +// The clip to read is the one the gains make as well as the sensor's own: on a body +// whose gains lift red and blue (the Ricoh GR: `cam_mul` [2.64, 1, 1.73]) a blown +// sky crosses the white level at 0.38 of the raw range in red and 0.58 in blue while +// green only crosses at 1.0, so a gate read on the sensor's levels alone stayed shut +// across the whole sky. Measured on that frame in the app against the camera's own +// JPEG: mean dRGB +1.2, -5.9, -6.1 with the sensor's clip alone, +0.8, +1.0, +1.1 +// with both, mean |dL| 21.5 against 11.1. +assert.match( + code, + /float hi = max\(max\(n\.r, n\.g\), n\.b\);\s*hi = max\(hi, max\(max\(lin\.r, lin\.g\), lin\.b\)\);\s*rgb = mix\(rgb, float3\(mx\), smoothstep\(0\.95, 1\.0, hi\)\);/ +); + +// The inverse is built on the CPU side and has to be the first uniform of the +// buffer the shader reads as `gain`, and its divisor has to be the white level +// the frame itself ran out at — `maximum` alone left every frame a stop bright, +// and a fixed factor two only fitted the two bodies it was measured on. The +// probe and its own check live in src/engine/sensorWhite.ts. +assert.match(src, /^ *uniforms\[0\] = SAMPLE_MAX \/ sensorWhite\(data, cd\.maximum, cd\.black\);$/m); +assert.match(src, /^import \{ sensorWhite \} from '\.\/sensorWhite';$/m); + +// On the FX30 that inverse is 2.065 (65535 / (2 (16380 - 512))), against the +// 4.13 the white level alone gives. The camera's own JPEG lands on the 2.065: +// 0.05% of its pixels at pure white against the develop's 0.76% at 4.13 — which +// is also what the probe has to fall back to on a frame with nothing blown. +assert.ok(65535 / (2 * (16380 - 512)) - 2.065 < 0.001, 'the white level factor dropped out'); +const probe = readFileSync(new URL('../src/engine/sensorWhite.ts', import.meta.url), 'utf8'); +assert.match( + probe, + /if \(top < maximum - black\) return legacy;/, + 'a frame with nothing blown no longer falls back to the fixed factor' +); + +// What a RAW opens as is the develop of its own sensor data, fitted to the +// preview the camera wrote into the file — so the file opens at the colour the +// body chose and at the resolution its sensor has. The preview is the reference +// and the fallback, never the frame: a file with no preview of its own is not +// fitted, and there is no body table behind the fit any more (a table fitted on +// one develop stops matching when the develop changes under it, and it did: the +// highlight knee left every body that had one with a cast). +assert.match(src, /preview = await cameraPreview\(raw\);/); +assert.match(src, /const match = blocks && ref \? fitMatch\(blocks, ref\) : null;/); +assert.doesNotMatch(src, /cameraMatch/, 'the body table is back'); +assert.doesNotMatch(code, /uniform float4 w0;/, 'the body table is back in the shader'); +// The fit is drawn by the develop that was fitted, so the frame goes through it +// twice: once on the sensor alone, to fit against the preview, and then again +// with the fit in the shader. +assert.match(src, /const first = develop\(null\);/); +assert.match(src, /const matched = match \? develop\(match\) : null;/); +assert.match(src, /const jpeg = \(matched \?\? first\)\.encodeToBytes/); +assert.match(src, /if \(preview\) return preview;/); +assert.match(src, /if \(thumb\?\.format !== 'jpeg' \|\| !thumb\.data\?\.length\) return null;/); + +// The fit has to land on the encoded value in float, not as an 8-bit colour filter +// painted over the frame afterwards: the fit carries an exposure, and a channel it +// lifted past the white level was cut where it stood — blue first, the channel the +// WB gains lift most — which is what left the bright end of the frame short of +// blue. Measured on the A5100 frame, pixels at 255: 5.25% blue, 2.10% of them blue +// alone through the filter, against 2.46% / 2.44% through this path. +// +// No rolloff after the fit. The fit is luma-preserving, not white-preserving — +// its rows sum to ~0.79 / 1.07 / 0.92, so white leaves it past 1.0 in green only — +// and dividing by that max pulled red and blue down with it: highlights came out +// cyan and not one cell of the frame reached white on all three channels (0.0%) +// against 2.3% now and 3.3% in the camera's preview, dE00 7.1 against 6.6. +assert.match(code, /uniform float4 f0;/); +assert.doesNotMatch(src, /drawMatched|colorMatrix/, 'the fit is painted through a colour filter again'); +assert.match(code, /float3 q = clamp\(float3\(dot\(f0\.xyz, e\), dot\(f1\.xyz, e\), dot\(f2\.xyz, e\)\), 0\.0, 1\.0\);/); +assert.doesNotMatch(code, /float mq = /, 'the rolloff after the fit is back'); + +// A 3x3 can only scale a channel and the gap to the camera is mostly a shape, so +// the frame leaves through the per-channel curve the fit carries as well. Without +// it the shadows stayed bright and green: measured on the A5100 frame, dE00 6.6 +// against 4.4, and at the bottom of the lightness range dL +13.8 with green +// +0.128 against +4.5 and +0.017 through the curve (both against the camera's own +// JPEG). +assert.match(code, /float tone\(float4 w, float x\) \{/); +assert.match(code, /return half4\(half3\(tone\(t0, q\.r\), tone\(t1, q\.g\), tone\(t2, q\.b\)\), 1\.0\);/); +assert.match(code, /uniform float4 t0;/); +// ...and it is the tail of the uniform buffer, one float4 per channel, which the +// second develop overwrites on the buffer it already built. +assert.match(src, /^ *uniforms\.set\(fit\?\.tone \?\? FLAT_TONE, 33\);$/m); +assert.match(src, /const uniforms = new Float32Array\(45\);/); +assert.match(src, /^ *uniforms\[21\] = f\[0\]; uniforms\[22\] = f\[1\]; uniforms\[23\] = f\[2\]; uniforms\[24\] = 0;$/m); + +// A RAW opens at the sensor's own resolution, not at the quarter the half-size +// demosaic reports: the GR's DNG came back 3010x2012 against the 6000x4000 of the +// camera's own JPEG beside it. With the flag off the same develop returns +// 6020x4024, and every one of the eight bodies checked doubled its frame. +assert.match(src, /^ *halfSize: false,$/m, 'the develop is half-size again'); + +console.log('raw-develop-check ok'); diff --git a/docker/frontend/scripts/white-level-check.mjs b/docker/frontend/scripts/white-level-check.mjs new file mode 100644 index 0000000..7c81b50 --- /dev/null +++ b/docker/frontend/scripts/white-level-check.mjs @@ -0,0 +1,67 @@ +// Where the develop's white level comes from (src/engine/sensorWhite.ts). The +// probe reads the frame instead of trusting a constant: the plane's ceiling is +// 1.89-2.00x `maximum - black` and which one depends on the body, and a frame +// that sat below the desaturation's 0.95 start came out tinted (Fuji X-Trans, +// measured 1.892). These pin the two states it has to tell apart — a blown +// plateau (spike plus gap) and a bright smooth sky (a top bin no bigger than the +// ones under it) — and the fall-backs on frames with nothing to measure. +// +// node scripts/white-level-check.mjs +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import ts from 'typescript'; + +const src = readFileSync(new URL('../src/engine/sensorWhite.ts', import.meta.url), 'utf8'); +const js = ts.transpileModule(src, { + compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2022 }, +}).outputText; +const { sensorWhite } = await import(`data:text/javascript,${encodeURIComponent(js)}`); + +const N = 1_000_000; +const MAXIMUM = 16383; +const BLACK = 512; +const LEGACY = 2 * (MAXIMUM - BLACK); +const FLOOR = 8192; // an unremarkable mid-tone the frame is mostly made of + +// A frame of `N` mid-tone samples plus `counts` of them sitting in the nine bins +// from top-8 up to top: counts[8] is the top bin, counts[0] the one eight below. +function plane(top, counts) { + const data = new Uint16Array(N); + data.fill(FLOOR); + let i = 0; + for (let d = 0; d < 9; d++) for (let k = 0; k < counts[d]; k++) data[i++] = top - 8 + d; + return data; +} + +// The sensor's own plateau: a pile at the top with nine near-empty bins under it. +assert.equal(sensorWhite(plane(30995, [1, 1, 1, 1, 1, 1, 1, 1, 3000]), MAXIMUM, BLACK), 30995, 'X-Trans plateau missed'); +assert.equal(sensorWhite(plane(31779, [1, 1, 1, 1, 1, 1, 1, 1, 3000]), MAXIMUM, BLACK), 31779, 'Sony plateau missed'); +// ...and the body that really does sit at twice the white level is read as such, +// not as the constant it used to be. Same answer either way, which is the point: +// the probe does not need to know which body it has. +assert.equal(sensorWhite(plane(LEGACY, [1, 1, 1, 1, 1, 1, 1, 1, 3000]), MAXIMUM, BLACK), LEGACY); + +// A smooth sky fills the bins around the top far more than a clipped one does, +// and its top bin is a tail, not a step: no gap, no clip, no move of the white. +assert.equal( + sensorWhite(plane(30995, [400, 900, 1500, 2000, 2400, 2600, 2600, 2400, 2000]), MAXIMUM, BLACK), + LEGACY, + 'a smooth top read as a clip' +); +// A bright flat wall sits exactly on one value with a big pile — but the bins +// under it are empty too, so that one IS a clip and is meant to move the white. +assert.equal(sensorWhite(plane(30995, [0, 0, 0, 0, 0, 0, 0, 0, 5000]), MAXIMUM, BLACK), 30995); + +// Nothing to measure: a frame that never reached the plane's ceiling, and one +// whose only pile is a handful of hot pixels, both stay on the fixed factor two. +assert.equal(sensorWhite(plane(9000, [1, 1, 1, 1, 1, 1, 1, 1, 3000]), MAXIMUM, BLACK), LEGACY, 'an unblown frame moved the white'); +assert.equal(sensorWhite(plane(30995, [0, 0, 0, 0, 0, 0, 0, 0, 5]), MAXIMUM, BLACK), LEGACY, 'hot pixels moved the white'); + +// The pile is a fraction of the plane, not a fixed count, so a small sensor with +// the same clipping is read the same way as a large one. +const small = new Uint16Array(100_000); +small.fill(FLOOR); +for (let i = 0; i < 40; i++) small[i] = 30995; +assert.equal(sensorWhite(small, MAXIMUM, BLACK), 30995, 'a small plane moved the threshold'); + +console.log('white-level-check ok'); diff --git a/docker/frontend/shared/utils/toneShader.ts b/docker/frontend/shared/utils/toneShader.ts index a4150c9..067a573 100644 --- a/docker/frontend/shared/utils/toneShader.ts +++ b/docker/frontend/shared/utils/toneShader.ts @@ -14,18 +14,28 @@ import { HSL_BANDS, hslBandGaps, isMonochromeBase } from './colorUtils'; // (a knee that started lower dragged a mid-grey down) while SH rides the lower // half (0.00..0.55), and the 0.50 midpoint never moves. // -// HL is Lightroom's highlight RECOVERY and it is weighted by the headroom that is -// left, (1 - t): the move falls to zero as t reaches pure white, so a blown sky, -// a lamp or a specular can be pulled back without the white going grey. The pull -// peaks around t = 0.79 at 0.13 of the ramp for a full -10 — the edge of a cloud, -// which is the detail there is to get back (measured, and checked for -// monotonicity over the whole -1..1 rectangle of the two knobs). +// HL is two different controls with one knob, because recovery and a lift are +// not the same move: // -// Both knobs are additive shifts of the luma. That keeps the curve monotonic -// (worst slope +0.003 at t = 0.99 with HL +10 and SH -10, and the two knees -// barely overlap), so a brighter input can never come out darker. The earlier -// multiplicative form was NOT monotonic: with hl=-1 a grey 0.73 came out -// darker than 0.80. +// -HL is Lightroom's highlight RECOVERY: a soft knee in LINEAR light over the +// top half (T = 0.5), pulled down by the ratio of the new luma to the old (see +// the knee in TONE_SKSL). That is the shape the doc asks for, and the shape the +// encoded domain cannot give — on the encoded value the last stop of headroom +// is a few code values wide. It is the only term here that is not a shift, so +// it is also the only one that can put detail back into a blown sky rather than +// merely darken it. T=0.5 and S = |hl| keep it monotone (the slope leaves the +// knee at 1 and falls, never rises) and it never brightens, so the frame cannot +// invert. +// +// +HL is a LIFT, weighted by the headroom that is left, (1 - t): the move falls +// to zero as t reaches pure white, so a lamp or a specular is not turned grey, +// and it rides into the upper midtones where a lift is wanted. The pull peaks +// around t = 0.79 at 0.13 of the ramp for a full +10. +// +// SH stays an additive shift. Shifts keep the curve monotonic (worst slope +// +0.003 at t = 0.99 with HL +10 and SH -10, and the two knees barely overlap), +// so a brighter input can never come out darker. The earlier multiplicative +// form was NOT monotonic: with hl=-1 a grey 0.73 came out darker than 0.80. // // dr - DR strength 0..1: lifts shadows slightly and rolls highlights // (Fuji extended DR); 0/auto/DR100 = no extra curve. @@ -137,9 +147,41 @@ float bandW(float hue, float anchor, float gapL, float gapR) { float d = mod(hue - anchor + 180.0, 360.0) - 180.0; return d <= 0.0 ? max(0.0, 1.0 + d / gapL) : max(0.0, 1.0 - d / gapR); } +// The sRGB transfer pair, the accurate one (0.04045/12.92 + 2.4) — the same +// constants colorUtils.planckianLinear uses on the WB side and EXPOSURE_SKSL +// uses for its own pass. Highlight recovery needs the same space: a knee drawn +// on the encoded value has the wrong shape (the midtones sit high and the last +// stop of headroom is squeezed into a few code values), which is exactly the +// doc's point about working in linear light. +vec3 toLinear(vec3 c) { + return mix(c / 12.92, pow((c + 0.055) / 1.055, vec3(2.4)), step(vec3(0.04045), c)); +} +vec3 toEncoded(vec3 c) { + return mix(c * 12.92, 1.055 * pow(c, vec3(1.0 / 2.4)) - 0.055, step(vec3(0.0031308), c)); +} vec4 main(vec2 xy) { vec4 c = src.eval(xy); vec3 rgb = clamp(c.rgb, 0.0, 1.0); + // HIGHLIGHT RECOVERY (-HL): the doc's soft knee, and it runs in LINEAR light, + // before the luma and the masks below are read, so every later stage sees the + // recovered value: + // L' = L , L < T + // L' = T + (L - T) / (1 + 2 S (L - T)) , L >= T + // with T = 0.5 and S the knob. The pixel is rebuilt by the ratio L'/L, so + // every channel keeps its share of the light and the hue and the saturation + // cannot drift; the curve leaves T with the slope it arrived with (1), so + // there is no seam at the knee; and the knee never brightens (S = 1 puts the + // white point on 0.75), which is what a recovery slider has to do. + if (hl < 0.0) { + vec3 lin = toLinear(rgb); + float l0 = dot(lin, vec3(0.2126, 0.7152, 0.0722)); + if (l0 > 0.5) { + float over = l0 - 0.5; + float S = min(-hl, 1.0); + lin *= (0.5 + over / (1.0 + S * over * 2.0)) / l0; + rgb = clamp(toEncoded(lin), 0.0, 1.0); + } + } float t = clamp(dot(rgb, vec3(0.2126, 0.7152, 0.0722)), 0.0, 1.0); float hlMask = smoothstep(0.50, 1.00, t); float shMask = 1.0 - smoothstep(0.00, 0.55, t); @@ -149,7 +191,11 @@ vec4 main(vec2 xy) { // weight is 0, so the pull-back cannot touch a pure white (a sun, a bulb, a // specular) and cannot turn it grey. A LIFT gets the same weight, which rides // it into the upper midtones and leaves the clipping where it was. - float o = t + hl * hlMask * (1.0 - t) + sh * 0.34 * shMask; + // The LIFT only, so max(hl, 0): -HL has already been spent in linear light + // above, and running it through this additive term as well would double-count + // it (and, being an additive shift, would darken the white the knee just + // protected). + float o = t + max(hl, 0.0) * hlMask * (1.0 - t) + sh * 0.34 * shMask; // Dynamic range: gentle shadow lift + highlight roll (protect brights). o += dr * 0.12 * shMask * (1.0 - t); o -= dr * 0.18 * hlMask * t; diff --git a/docker/frontend/src/engine/cameraMatch.ts b/docker/frontend/src/engine/cameraMatch.ts deleted file mode 100644 index 56d7ba6..0000000 --- a/docker/frontend/src/engine/cameraMatch.ts +++ /dev/null @@ -1,67 +0,0 @@ -// The look a camera's own JPEG would have had. -// -// LibRaw here is deliberately kept out of white balance and tone (see -// rawDevelop), so a RAW that opens in the studio lands on the neutral demosaic — -// while the JPEG the photographer saw on the back of the camera carries the -// body's own colour rendering. These 3x3 matrices are that difference, fitted -// offline: develop a frame through this same path, compare it block by block -// against the camera's own preview of the same frame, and least-squares the -// matrix that takes the first to the second. Fitted per body on real scenes -// (4–12 MP each, held-out blocks scored with CIEDE2000): -// -// Ricoh GR III 4.28 (unfitted 5.99) Fujifilm X100V 4.22 (8.19) -// Ricoh GR II 9.73 (11.68) Fujifilm X100S 7.27 (7.41) -// Fujifilm X-T3 6.12 (9.03) -// -// What is left over is largely high-frequency (sharpening, noise reduction, -// demosaic) rather than colour: the error keeps falling as the blocks grow -// (GR III: 2.86 at 6px, 2.06 at 24px, 1.59 at 60px). -// -// The transform preserves luma exactly — out = luma * normalize(M * in / luma) — -// so it moves colour and never exposure. A preview that came out dark stays -// dark, on purpose: exposure is the studio's job, not the profile's. ponytail: -// one matrix, no tone curve and no 3D LUT; a curve on top was measured at 2% -// better and needs a spline plus an array uniform, so add one only when a body -// turns out to need it. -// -// Keyed on LibRaw's normalized_model — the bare model ("GR III", "X100V"), not -// the make-qualified camera_model: a body that was never fitted, or that reports -// a name we do not know, develops exactly as it did before. -export type Mat3 = readonly [ - number, number, number, - number, number, number, - number, number, number, -]; - -// Row-major, applied to the sRGB-encoded develop (the space it was fitted in). -const MATCH: Record = { - 'GR III': [ - 0.577754, 0.856314, -0.440638, - 0.143396, 0.716041, 0.145206, - -0.177111, 0.291345, 0.859118, - ], - 'GR II': [ - 0.440692, 0.976166, -0.377622, - 0.187633, 0.680171, 0.121235, - -0.211717, 0.293754, 0.911013, - ], - 'X-T3': [ - 1.626754, -0.533332, -0.130089, - -0.184760, 1.160799, 0.039331, - -0.015344, -0.022394, 0.993452, - ], - 'X100V': [ - 1.103373, 0.362157, -0.471229, - 0.014568, 0.833165, 0.154949, - -0.448700, 0.586230, 0.852687, - ], - 'X100S': [ - 0.527629, 0.469773, -0.014534, - 0.144123, 0.823023, 0.033459, - -0.036716, 0.369805, 0.711359, - ], -}; - -export function cameraMatch(model?: string | null): Mat3 | null { - return (model && MATCH[model]) || null; -} diff --git a/docker/frontend/src/engine/previewMatch.ts b/docker/frontend/src/engine/previewMatch.ts new file mode 100644 index 0000000..9703591 --- /dev/null +++ b/docker/frontend/src/engine/previewMatch.ts @@ -0,0 +1,186 @@ +// The colour the camera itself wrote into the file. +// +// LibRaw hands back the sensor's own data and nothing about how the body chose +// to render it: the white balance it locked, its picture style, its tone curve. +// The file carries the answer anyway — the preview saved inside it, which is the +// frame the photographer saw and the one a desktop viewer shows. So the develop +// is matched to that preview: develop a grid of the frame, take the same grid out +// of the embedded preview, and least-squares the 3x3 that takes the first to the +// second — then, on what that 3x3 leaves, the tone curve a matrix cannot hold. +// +// Measured on the A5100 frame the studio was reported on (dE00 against the +// camera's own 24MP JPEG, blocks of a 64x64 grid): 12.0 as the develop left it, +// 9.2 with this fit drawn in linear light, 6.4 painted through the 8-bit colour +// filter this used to be, 6.6 with the 3x3 alone in float, 4.4 with the tone +// curve under it. What is left either way is the camera's own local rendering — +// its sharpening, the detail a 1.7MP preview never had — and no per-pixel +// transform reaches that. +// +// The fit is on the sRGB-encoded values, exposure included — matrix and curve +// alike: the reference is the file's own frame, so matching its brightness is +// part of matching its colour. Fitted in that encoded space, not in linear light, +// because a least squares in linear light weighs the highlights and comes back +// with the midtones wrong — the same frames scored dE00 6.4 fitted encoded +// against 9.2 fitted linear. Any body, any picture style, and no fitted table to +// go stale — a preview is the one profile every RAW already carries. +// Row-major 3x3, on the sRGB-encoded develop: what the shader applies to a pixel. +export type Mat3 = readonly [ + number, number, number, + number, number, number, + number, number, number, +]; + +// The grid the fit reads, and the size of the preview map handed to it. +export const MATCH_GRID = 64; + +// A block at or past this has nothing left to say about a rendering. +const CLIP = 0.97; +const FLOOR = 0.03; + +// The per-file colour fit: a 3x3, then the tone curve a 3x3 cannot hold — one +// cubic a channel, on the encoded value the 3x3 leaves. `tone` is that curve in +// the shader's own order: r a b c d, then g, then b. +export interface Match { + m: Mat3; + tone: Float32Array; +} + +// What the second stage does when the frame has no curve the fit can trust. +export const FLAT_TONE = new Float32Array([0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0]); + +// Gaussian elimination on the one system this file fits: a channel is a row of +// three, for the 3x3 and again for the 3x3's own error. +function solve(C: number[][], d: number[]): number[] { + const n = 3; + const a = Array.from({ length: n }, (_, i) => [...C[i], d[i]]); + for (const row of a) for (let c = 0; c < n; c++) row[c] += 1e-9; + for (let c = 0; c < n; c++) { + let p = c; + for (let r = c + 1; r < n; r++) if (Math.abs(a[r][c]) > Math.abs(a[p][c])) p = r; + [a[c], a[p]] = [a[p], a[c]]; + for (let r = 0; r < n; r++) { + if (r === c) continue; + const f = a[r][c] / a[c][c]; + for (let k = c; k <= n; k++) a[r][k] -= f * a[c][k]; + } + } + return a.map((_, i) => a[i][n] / a[i][i]); +} + +// The curve a channel needs to reach the camera's, least squared: what the 3x3 +// left for that channel, a cubic against the block the camera wrote there. +// +// The cubic is pinned at white, not free. Fitted free it slid its far end down to +// meet the blocks it wanted — 0.96 on red — and every blown highlight the develop +// had carried up to the white level came out off-white instead: a blown pixel +// (1, 1, 1) into the fit left at 0.82 / 0.97 / 0.93, and 0.0% of the frame at +// pure white against 2.3% before and 3.3% in the preview. So the fit solves the +// three free terms of `tone = x^3 + a(1 - x^3) + b(x - x^3) + c(x^2 - x^3)`, whose +// terms all vanish at 1 and which costs the fit almost nothing (rms 0.066 on red +// against the free cubic's 0.063, and 0.076 on blue against 0.076, on the frame +// this was measured on). Scaling the free curve onto its own end instead moved +// the midtones by the whole 4% the end was short, and pinning both ends moved them +// by 12% (rms 0.21 on green). +function basis(i: number, x: number): number { + const x3 = x * x * x; + return i === 0 ? 1 - x3 : i === 1 ? x - x3 : x * x - x3; +} + +function fitTone(blocks: { t: number[]; q: number[] }[]): Float32Array { + const out = new Float32Array(12); + for (let c = 0; c < 3; c++) { + const A = [[0, 0, 0], [0, 0, 0], [0, 0, 0]]; + const d = [0, 0, 0]; + for (const { t, q } of blocks) { + const x = q[c], b = [basis(0, x), basis(1, x), basis(2, x)]; + for (let p = 0; p < 3; p++) { + for (let k = 0; k < 3; k++) A[p][k] += b[p] * b[k]; + d[p] += b[p] * (t[c] - x * x * x); + } + } + const [a, b, cc] = solve(A, d); + out.set([a, b, cc, 1 - a - b - cc], c * 4); + } + return out; +} + +// A curve that comes back folded, or wandered far from the value it was handed, +// is a fit off blocks that fought each other, not a rendering: the caller keeps +// the 3x3 alone rather than a look nobody's camera has. The floor sits below the +// zero a curve is allowed to reach — the develop clamps there anyway — and the +// slope is only asked not to turn back. +function curveIsSane(tone: Float32Array): boolean { + for (const v of tone) if (!Number.isFinite(v) || Math.abs(v) > 4) return false; + for (let c = 0; c < 3; c++) { + const [a, b, cc, dd] = [tone[c * 4], tone[c * 4 + 1], tone[c * 4 + 2], tone[c * 4 + 3]]; + let prev = a; + for (let i = 1; i <= 16; i++) { + const x = i / 16; + const y = a + x * (b + x * (cc + x * dd)); + if (y < prev - 0.12 || y < -0.15 || y > 1.15) return false; + prev = y; + } + } + return true; +} + +// `dev` and `ref` are n x n RGBA maps of the same frame, one byte per channel. +// The 3x3 comes back on those encoded values — the space the caller draws it in. +// Null when the two cannot be related — too few blocks carry colour, or the +// solution runs away — and the caller develops the frame as the sensor left it. +export function fitMatch(dev: Uint8Array, ref: Uint8Array, n = MATCH_GRID): Match | null { + const C = [[0, 0, 0], [0, 0, 0], [0, 0, 0]]; + const d = [[0, 0, 0], [0, 0, 0], [0, 0, 0]]; + const blocks: { e: number[]; t: number[] }[] = []; + let used = 0; + for (let i = 0; i < n * n; i++) { + const e = [dev[i * 4] / 255, dev[i * 4 + 1] / 255, dev[i * 4 + 2] / 255]; + const t = [ref[i * 4] / 255, ref[i * 4 + 1] / 255, ref[i * 4 + 2] / 255]; + // A block the camera clipped has no colour left to copy, one at the floor is + // only the pedestal, and one the develop clipped has lost the ratio between + // its channels — none of them say anything about the rendering. + if (t[0] > CLIP && t[1] > CLIP && t[2] > CLIP) continue; + if (e[0] > CLIP || e[1] > CLIP || e[2] > CLIP) continue; + if (Math.max(t[0], t[1], t[2]) < FLOOR) continue; + used++; + blocks.push({ e, t }); + for (let x = 0; x < 3; x++) { + for (let y = 0; y < 3; y++) C[x][y] += e[x] * e[y]; + for (let c = 0; c < 3; c++) d[x][c] += e[c] * t[x]; + } + } + // A fit off a handful of blocks is a fit off noise. + if (used < (n * n) / 8) return null; + // Ridge towards identity, scaled to the data. The three channels of a develop + // are near-collinear — a camera's own rendering of a neutral scene is close to + // neutral — so the plain normal equations come back singular-ish, and the frame's + // own noise decides a matrix whose diagonal lands at or below zero. λ of 1e-3 of + // the data's scale lifts that back to a definite system and moves the fitted + // matrix only in the third decimal (measured: 0.001 kept the tone the same to + // 0.3% where 0.1 began flattening it). + const lambda = 1e-3 * ((C[0][0] + C[1][1] + C[2][2]) / 3); + for (let r = 0; r < 3; r++) { + C[r][r] += lambda; + d[r][r] += lambda; + } + const m = [...solve(C, d[0]), ...solve(C, d[1]), ...solve(C, d[2])]; + // A row that ran off is a degenerate fit, not a look. A diagonal element at or + // below zero is not one: what these matrices hold is the difference between two + // near-identical channels (X-T3 -0.04, X100V -1.67 on a row that carries +1.05 + // and -1.67), and refusing those refused the whole fit — those bodies opened + // byte-identical to an unfitted develop. + for (const v of m) if (!Number.isFinite(v) || Math.abs(v) > 4) return null; + // The 3x3 alone cannot bend a channel — it can only scale it — and what is left + // between the two frames is mostly exactly that bend. So the residual after the + // 3x3 is fitted a cubic a channel, on the value the 3x3 leaves. + const q = blocks.map(({ e, t }) => { + const v = [ + m[0] * e[0] + m[1] * e[1] + m[2] * e[2], + m[3] * e[0] + m[4] * e[1] + m[5] * e[2], + m[6] * e[0] + m[7] * e[1] + m[8] * e[2], + ]; + return { t, q: v.map((x) => Math.min(1, Math.max(0, x))) }; + }); + const tone = fitTone(q); + return { m: m as unknown as Mat3, tone: curveIsSane(tone) ? tone : FLAT_TONE }; +} diff --git a/docker/frontend/src/engine/rawDevelop.ts b/docker/frontend/src/engine/rawDevelop.ts index 4388568..d8729a3 100644 --- a/docker/frontend/src/engine/rawDevelop.ts +++ b/docker/frontend/src/engine/rawDevelop.ts @@ -1,15 +1,49 @@ // RAW → JPEG on the client, so a camera's own file opens in the studio without // a DNG converter in the middle (see native_raw_processing_opfs_architecture.md). // -// LibRaw demosaics in its own worker; what comes back is linear camera data, +// A RAW opens at the colour the camera chose for it. LibRaw is kept out of white +// balance and tone (below) and knows nothing of the body's picture style, so the +// develop is fitted to the one rendering the file does carry with it: the preview +// the camera wrote inside it, which is the frame the photographer saw and the one +// a desktop viewer shows. Develop the sensor, least-squares a 3x3 from a block +// grid of each onto the other on the encoded values, fit what that 3x3 leaves a +// cubic a channel, and develop the sensor through both (see previewMatch.ts). +// +// Measured against the camera's own 24MP JPEG of the A5100 frame this was +// reported on, blocks of a 64x64 grid, dE00: 12.0 as the develop left it, 19.7 +// with no white balance at all, 6.6 matched to the file's own preview by the 3x3 +// alone and 4.4 through the 3x3 and the curve together. What neither reaches is +// the camera's own sharpening, which no per-pixel transform holds. +// +// The fit is the only profile: a body-used-to-be table of six matrices fitted on +// this develop was dropped, because a table fitted on one path stops matching the +// moment the path changes under it — its rows no longer summed to 1 once the +// highlight knee landed ahead of it, and every body that had one opened with a +// cast the body that had none did not. Offline-fitted matrices age; the file's +// own preview does not. +// +// Developing the sensor is also what keeps the highlights: 2.3% of the frame at +// pure white against 3.3% in the preview itself, since the 8-bit preview threw +// the headroom away and the develop still has it. The preview is therefore the +// reference and the fallback, not the frame: it is what a file with no usable +// preview (some DNG), or with sensor data that will not decode, opens as. +// +// The develop: LibRaw demosaics in its own worker; what comes back is linear camera data, // which this file turns into the sRGB the rest of the pipeline expects. The -// whole thing is measured against a real 26MP Sony ARW — the settings below are -// the ones that gave the correct colours there: +// settings below were checked against the preview a Sony ILME-FX30 writes into +// its own ARW (the camera JPEG, read straight out of the file): the developed +// frame and that preview agree to within 1% on both channel ratios, R/G 0.909 +// against 0.903 and B/G 0.603 against 0.606. // - noAutoScale + useCameraWb:false + noAutoBright + gamm [1,1] keep LibRaw out // of white balance and tone, so `cam_mul` and `rgb_cam` can be applied here // exactly once. -// - halfSize: 26MP → 6.5MP. ponytail: drop it for full resolution if a user -// ever asks for a print from the RAW; the develop pass is the whole cost. +// - halfSize:false: the frame opens at the sensor's own resolution — a 24MP +// RAW develops to 24MP (6020x4024 on the GR, 6000x4000 on the Fuji), not to +// the quarter the half-size demosaic reports. `userQual` 3 then demosaics +// all of it. The develop was half-size until a GR's DNG opened at 3010x2012 +// against its own JPEG's 6000x4000: the quarter-size frame was the flag, +// not the file. ponytail: costs ~4x the develop time and two full-size F16 +// surfaces; put the flag behind a "draft" toggle if a phone ever has to. // // The band loop exists because a single Float32 copy of the whole plane would be // ~100MB. Each band is decoded, normalised and drawn before the next is read. @@ -17,14 +51,24 @@ // the main thread (Skia is not available in the RAW worker). Move it to a worker // with an OffscreenCanvas if the develop ever blocks the UI visibly. import LibRaw from 'libraw-wasm'; -import { cameraMatch, type Mat3 } from './cameraMatch'; +import { sensorWhite } from './sensorWhite'; import { f32ToF16 } from './halfFloat'; +import { fitMatch, FLAT_TONE, MATCH_GRID, type Match, type Mat3 } from './previewMatch'; import { Skia } from './skiaShim'; -// What `imageData()` returns for the settings below: 16-bit, 3 channels, with -// the black level still in it — hence the two normalisations in the shader. +// What `imageData()` returns for the settings below: 16-bit, 3 channels, black +// level already gone — the post-process subtracts it whatever `noAutoScale` +// says, which only holds back the white balance and the output scaling. +// +// Its white level is not `maximum` but the frame's own plateau, a factor of 1.89 +// to 2.00 out (see sensorWhite below). The gain carries that level, so the white +// lands back on 1.0 — and on every body, not just the two that factor two was +// fitted on: without it every frame opened a stop bright (the FX30's own JPEG has +// 0.05% of pixels at pure white where the develop had 0.76%) and a body that sat +// lower (X-Trans, 1.892) never even reached the highlight desaturation, which +// starts at 0.95 of the sensor. const SETTINGS = { - halfSize: true, + halfSize: false, outputBps: 16, outputColor: 0, noAutoScale: true, @@ -46,53 +90,103 @@ const IDENTITY: Mat3 = [1, 0, 0, 0, 1, 0, 0, 0, 1]; const RAW_DEVELOP_SKSL = ` uniform shader raw; -uniform float4 black; // (black level, 1 / (white level - black level)) +uniform float gain; // 1 / the white level the frame itself ran out at uniform float4 mul; // cam_mul, green-normalised uniform float4 m0; // camera -> sRGB, the first three columns of rgb_cam uniform float4 m1; uniform float4 m2; uniform float4 crop; // (y offset of this band, 0, 0, 0) -uniform float4 w0; // the body's camera match, one row per float4 -uniform float4 w1; // (identity when the body has no profile) -uniform float4 w2; +uniform float4 f0; // the per-file fit, on the encoded value (identity when +uniform float4 f1; // the frame carries no preview to be fitted to) +uniform float4 f2; +uniform float4 t0; // what that fit then leaves a channel: one cubic a +uniform float4 t1; // channel, r a b c d per float4 (flat when there is no fit) +uniform float4 t2; + +// A 3x3 can only scale a channel; the gap to the camera is mostly a shape, and +// this is that shape, read on the value the 3x3 left. +float tone(float4 w, float x) { + return clamp(w.x + x * (w.y + x * (w.z + x * w.w)), 0.0, 1.0); +} float3 encode(float3 x) { x = clamp(x, 0.0, 1.0); return mix(x * 12.92, 1.055 * pow(x, float3(1.0 / 2.4)) - 0.055, step(float3(0.0031308), x)); } -float luma(float3 x) { - return dot(float3(0.2126, 0.7152, 0.0722), x); -} - half4 main(float2 pos) { float4 p = raw.eval(float2(pos.x, pos.y - crop.x)); - // Only the floor here. A photo the sensor could not hold goes over the white + // The plane arrives with the black level already subtracted — it floors at 0, + // not at color_data.black (measured on the FX30 ARW: the sensor mosaic floors + // at 334, the plane at 0, a quarter of its red samples under the black level). + // Subtracting it again drained red and blue — the two channels the gains lift + // most — and dragged every frame towards green. + float3 n = max(p.rgb * gain, 0.0); // the sensor's own levels, white level 1.0 + float3 lin = n * mul.rgb; + // Only the floor there. A photo the sensor could not hold goes over the white // level in all three channels, and clipping them one by one before the WB gains // is what tints what is left of the highlight: green — the channel the gains are // normalised to — stops at 1.0 while red and blue, which need their 2.6x and // 1.6x, are already past it, so the blown area comes out magenta. Keep the // channel ratios through the matrix instead and let the overflow fade to white. - float3 lin = max((p.rgb - black.rgb) * black.a, 0.0) * mul.rgb; float3 rgb = float3(dot(m0.xyz, lin), dot(m1.xyz, lin), dot(m2.xyz, lin)); float mx = max(max(rgb.r, rgb.g), rgb.b); - // The overflow fades towards white instead of being cut, so the trace of a - // sensor that ran past its white level survives as a compressed ramp — which is - // what LIGHT's HIGHLIGHT row then has to pull on. ponytail: the ramp lives in - // [0.5, 1] and the band leaves as 8-bit JPEG, so the two stops above white are - // compressed, not kept; give develop a 16-bit output (or a knee of its own) when - // RAW highlights have to be recovered rather than merely look right. - rgb = mx > 1.0 ? mix(rgb / mx, float3(1.0), 1.0 - 1.0 / mx) : rgb; + // A pixel that has run to the white level has no colour of its own left to keep, + // and what the gains made of it is an artefact, not a colour: ease the pixel + // towards the neutral of its own value as that point is approached. The clip to + // read is the one the gains make, not only the sensor's own — the gains here are + // 1.7x and 1.9x on red and blue, so a blown sky reaches the white level at 0.59 of + // the raw range in those channels while the green, which the gains are + // normalised to, only reaches it at 1.0. Read on the sensor's levels alone the + // gate stayed shut across a whole blown sky and left the develop's magenta in it + // (227,184,245 at the gate's own value against 245,245,245 read where the gains + // put the clip, the camera's preview white at that block). Both are read, so a + // body whose gains do not lift a channel keeps the sensor's own clip as its gate. + // The matrix was fitted luma-preserving, so mx is the value to hold. + float hi = max(max(n.r, n.g), n.b); + hi = max(hi, max(max(lin.r, lin.g), lin.b)); + rgb = mix(rgb, float3(mx), smoothstep(0.95, 1.0, hi)); + // The overflow used to fade towards white — mix(rgb / mx, 1, 1 - 1 / mx) — + // which put every pixel of a blown sky on exactly 1.0 and threw the two stops + // the sensor held above white away with it: LIGHT's HIGHLIGHT row then had a + // flat white to pull on and nothing to reveal. The white point is moved down + // instead and the overflow squeezed back in under it by the doc's soft knee: + // + // y = T + over / (1 + 2S*over), over = mx - T, S = 1 / (2(1 - T)) + // + // S is what puts 1.0 on the asymptote, so the sensor's own plateau — two white + // levels up, see the gain above — lands at ~0.94 and the first stop over white + // spends 0.85..0.94. Below T the frame is untouched and the curve leaves T with + // the slope it arrived with (1), so there is no seam to mask; above it the frame + // darkens, which is the one move no later pass can undo — which is the point, + // and what the per-file fit below then measures the REST of the frame back from. + // + // ponytail: 4.7 stops of headroom now share ~5% of the ramp, and the develop + // still leaves as an 8-bit JPEG. Give it a float16 output when RAW highlights + // have to print rather than merely be seen. + if (mx > 0.7) { + float over = mx - 0.7; + rgb *= (0.7 + over / (1.0 + over * 3.3333)) / mx; + } float3 e = encode(rgb); - // The body's colour, in the encoded space it was fitted in. The matrix was - // fitted luma-preserving, and the luma(e) / luma(c) factor holds that exact: - // the profile turns the chroma and never rescues an exposure, which stays the - // studio's job. Identity is a no-op. - float3 c = clamp(float3(dot(w0.xyz, e), dot(w1.xyz, e), dot(w2.xyz, e)), 0.0, 1.0); - // ponytail: the band leaves as 8-bit JPEG, so the match is applied to a value - // that is already quantised — plenty for a look, but give develop 16-bit output - // if a profile ever has to grade rather than merely match. - return half4(half3(clamp(c * (luma(e) / max(luma(c), 1e-4)), 0.0, 1.0)), 1.0); + // The file's own colour: the fit, in float, on the encoded value it was fitted + // on — and the last step the frame leaves through. Not the 8-bit colour filter + // this used to be painted through: the fit carries an exposure (the preview is + // the reference, so matching its brightness is part of matching its colour). + // No rolloff here either. The fit is fitted whole to the preview's own values, + // and white is one of them, so it already maps the frame's white to the preview's + // — while a divide by the row max is a white-preserving move the fit does not + // need: every highlight came back at ~0.74 / 1.0 / 0.86, cyan, and not one cell + // of the frame reached white on all three channels (0.0% against the preview's + // 3.3%). Past the white level only the clamp is left, exactly as the develop + // above lets its own overflow run. + float3 q = clamp(float3(dot(f0.xyz, e), dot(f1.xyz, e), dot(f2.xyz, e)), 0.0, 1.0); + // ...and then the curve, one cubic a channel, which is what carries the body's + // own tone: a 3x3 can only scale, so the frame without it came back bright and + // green in the shadows (dL +13.8 and green +0.128 at the bottom of the range, + // dE00 6.6) instead of matching (dL +4.5, green +0.017, 4.4). The curve is + // pinned at white, so a blown pixel still lands on white. + return half4(half3(tone(t0, q.r), tone(t1, q.g), tone(t2, q.b)), 1.0); } `; @@ -110,11 +204,71 @@ export function isRawName(name: string): boolean { return name.includes('.') && RAW_EXT.includes(ext); } +// The camera's own preview, when the file carries one: the colour reference to +// fit against, and what the file opens as when the sensor does not decode. +// ponytail: it is 1616x1080 on an A5100 and 1620x1080 on an FX30, so handing it +// back is a 1.7MP frame — a print past it has to come off the develop, which is +// what a fitted file already gives. +async function cameraPreview(raw: LibRaw): Promise { + const thumb = await raw.thumbnailData().catch(() => undefined); + if (thumb?.format !== 'jpeg' || !thumb.data?.length) return null; + return new Uint8Array(thumb.data); +} + +// MATCH_GRID x MATCH_GRID block colours of a frame, one byte per channel. +function gridOf(image: any, n = MATCH_GRID): Uint8Array | null { + const surface = Skia.Surface.MakeOffscreen(n, n) ?? Skia.Surface.Make(n, n); + if (!surface) return null; + const canvas = surface.getCanvas(); + // Cubic, not a linear tap: this is a 45x reduction and linear reads a handful + // of source pixels per block — noise for the least squares to fit. + canvas.drawImageRectCubic( + image, + Skia.XYWHRect(0, 0, image.width(), image.height()), + Skia.XYWHRect(0, 0, n, n), + 1 / 3, + 1 / 3 + ); + surface.flush(); + const px = canvas.readPixels(0, 0, { + width: n, + height: n, + colorType: Skia.ColorType.RGBA_8888, + alphaType: Skia.AlphaType.Unpremul, + colorSpace: Skia.ColorSpace.SRGB, + }) as Uint8Array | null; + surface.dispose(); + return px ? new Uint8Array(px.buffer, px.byteOffset, px.byteLength) : null; +} + +// The same grid out of the preview, which the file carries as a JPEG. Decoded and +// reduced through the same Skia call as the develop's own grid, because the two +// grids are only comparable — and the fit only meaningful — when one resampler +// made both. A 2D canvas here instead left the fit following its own smoothing: +// on the A5100 frame the same develop scored dE00 4.7 against 4.4, and the dark +// end of the frame came out 6 L further from the preview than the fit it was +// handed asked for. +function previewGrid(jpeg: Uint8Array, w: number, h: number, n = MATCH_GRID): Uint8Array | null { + const bmp = Skia.Image.MakeImageFromEncoded(jpeg); + if (!bmp) return null; + try { + // A preview of another shape is a crop of the frame, not the frame: fitting + // against it lines the two grids up on different scenes and fits nothing. + if (Math.abs(bmp.width() / bmp.height() / (w / h) - 1) > 0.02) return null; + if (bmp.width() < n * 4) return null; + return gridOf(bmp, n); + } finally { + bmp.delete(); + } +} + export async function developRaw(bytes: Uint8Array): Promise { const raw = new LibRaw(); + let preview: Uint8Array | null = null; try { // LibRaw copies the buffer it is handed, so the caller's bytes stay intact. await raw.open(bytes as unknown as BufferSource, SETTINGS); + preview = await cameraPreview(raw); const meta = await raw.metadata(true); const img = await raw.imageData(); const cd = meta?.color_data; @@ -124,8 +278,6 @@ export async function developRaw(bytes: Uint8Array): Promise { const data = img.data as Uint16Array; if (!w || !h) throw new Error('RAW decoded to nothing'); - const surface = Skia.Surface.MakeOffscreen(w, h) ?? Skia.Surface.Make(w, h); - if (!surface) throw new Error('no surface for the develop'); const effect = Skia.RuntimeEffect.Make(RAW_DEVELOP_SKSL); if (!effect) throw new Error('develop shader failed to compile'); @@ -133,62 +285,91 @@ export async function developRaw(bytes: Uint8Array): Promise { const mul = cd.cam_mul.map((v) => v / green); const row = (i: number) => cd.rgb_cam[i].slice(0, 3); const [r0, r1, r2] = [row(0), row(1), row(2)]; - const cm: Mat3 = cameraMatch(meta?.normalized_model) ?? IDENTITY; const bandH = Math.max(1, Math.min(h, Math.floor(BAND_PIXELS / w))); const f32 = new Float32Array(w * bandH * 4); // The band goes up as half, not float32: the GPU backend puts an F32 image // on the 1/255 grid and the shadows quantise to black (see halfFloat.ts). const half = new Uint16Array(w * bandH * 4); - for (let y0 = 0; y0 < h; y0 += bandH) { - const rows = Math.min(bandH, h - y0); - let o = 0; - for (let i = y0 * w * 3, end = (y0 + rows) * w * 3; i < end; i += 3) { - f32[o++] = data[i] / SAMPLE_MAX; - f32[o++] = data[i + 1] / SAMPLE_MAX; - f32[o++] = data[i + 2] / SAMPLE_MAX; - f32[o++] = 1; - } - f32ToF16(f32, half, w * rows * 4); - const band = Skia.Image.MakeImage( - { width: w, height: rows, colorType: Skia.ColorType.RGBA_F16, alphaType: Skia.AlphaType.Unpremul }, - new Uint8Array(half.buffer, 0, w * rows * 8), - w * 8 - ); - if (!band) throw new Error('band image failed'); - const child = band.makeShaderOptions( - Skia.TileMode.Clamp, - Skia.TileMode.Clamp, - Skia.FilterMode.Nearest, - Skia.MipmapMode.None - ); - const uniforms = new Float32Array([ - cd.black / SAMPLE_MAX, cd.black / SAMPLE_MAX, cd.black / SAMPLE_MAX, - SAMPLE_MAX / (cd.maximum - cd.black), - mul[0], mul[1], mul[2], 0, - r0[0], r0[1], r0[2], 0, - r1[0], r1[1], r1[2], 0, - r2[0], r2[1], r2[2], 0, - y0, 0, 0, 0, - cm[0], cm[1], cm[2], 0, - cm[3], cm[4], cm[5], 0, - cm[6], cm[7], cm[8], 0, - ]); - const shader = effect.makeShaderWithChildren(uniforms, [child]); - const paint = Skia.Paint(); - paint.setShader(shader); - surface.getCanvas().drawRect(Skia.XYWHRect(0, y0, w, rows), paint); - surface.flush(); - paint.delete(); - shader.delete(); - child.delete(); - band.delete(); - } + // Every uniform but the crop and the fit, which are the two the passes change: + // the shader's own order is gain, mul, rgb_cam, crop, the fit, then the fit's + // tone curve. + const uniforms = new Float32Array(45); + uniforms[0] = SAMPLE_MAX / sensorWhite(data, cd.maximum, cd.black); + uniforms.set([mul[0], mul[1], mul[2], 0, r0[0], r0[1], r0[2], 0, r1[0], r1[1], r1[2], 0, r2[0], r2[1], r2[2], 0], 1); - const jpeg = surface.makeImageSnapshot().encodeToBytes(Skia.ImageFormat.JPEG, 92); - surface.dispose(); + // One develop of the frame, band by band, through `fit` when there is one. A + // function because the frame is drawn twice: once on the sensor alone, to fit + // against the preview, and then again with the fit in the shader. + // ponytail: two full band passes, on the main thread. Give develop an F16 + // intermediate (one develop, one colour pass) if the second pass ever shows. + const develop = (fit: Match | null) => { + const surface = Skia.Surface.MakeOffscreen(w, h) ?? Skia.Surface.Make(w, h); + if (!surface) return null; + const f = fit?.m ?? IDENTITY; + uniforms[21] = f[0]; uniforms[22] = f[1]; uniforms[23] = f[2]; uniforms[24] = 0; + uniforms[25] = f[3]; uniforms[26] = f[4]; uniforms[27] = f[5]; uniforms[28] = 0; + uniforms[29] = f[6]; uniforms[30] = f[7]; uniforms[31] = f[8]; uniforms[32] = 0; + uniforms.set(fit?.tone ?? FLAT_TONE, 33); + for (let y0 = 0; y0 < h; y0 += bandH) { + const rows = Math.min(bandH, h - y0); + let o = 0; + for (let i = y0 * w * 3, end = (y0 + rows) * w * 3; i < end; i += 3) { + f32[o++] = data[i] / SAMPLE_MAX; + f32[o++] = data[i + 1] / SAMPLE_MAX; + f32[o++] = data[i + 2] / SAMPLE_MAX; + f32[o++] = 1; + } + f32ToF16(f32, half, w * rows * 4); + const band = Skia.Image.MakeImage( + { width: w, height: rows, colorType: Skia.ColorType.RGBA_F16, alphaType: Skia.AlphaType.Unpremul }, + new Uint8Array(half.buffer, 0, w * rows * 8), + w * 8 + ); + if (!band) throw new Error('band image failed'); + const child = band.makeShaderOptions( + Skia.TileMode.Clamp, + Skia.TileMode.Clamp, + Skia.FilterMode.Nearest, + Skia.MipmapMode.None + ); + uniforms[17] = y0; + const shader = effect.makeShaderWithChildren(uniforms, [child]); + const paint = Skia.Paint(); + paint.setShader(shader); + surface.getCanvas().drawRect(Skia.XYWHRect(0, y0, w, rows), paint); + surface.flush(); + paint.delete(); + shader.delete(); + child.delete(); + band.delete(); + } + + const shot = surface.makeImageSnapshot(); + surface.dispose(); + return shot; + }; + + // The file's own colour: a grid of the develop as it stands, the same grid out + // of the preview the camera wrote into the file, and the 3x3 and the curve + // between them — which the second develop then draws in the shader. No + // preview, no fit: the frame opens as the sensor left it. + const first = develop(null); + if (!first) throw new Error('no surface for the develop'); + const blocks = preview ? gridOf(first) : null; + const ref = blocks ? previewGrid(preview as Uint8Array, first.width(), first.height()) : null; + const match = blocks && ref ? fitMatch(blocks, ref) : null; + const matched = match ? develop(match) : null; + const jpeg = (matched ?? first).encodeToBytes(Skia.ImageFormat.JPEG, 92); + (matched ?? first).dispose(); + if (matched) first.dispose(); if (!jpeg?.length) throw new Error('develop produced no bytes'); return jpeg; + } catch (err) { + // The preview still opens the file when the sensor will not: a RAW whose + // colour data is missing (some DNG) is not a RAW that cannot be shown. + if (preview) return preview; + throw err; } finally { raw.dispose(); } diff --git a/docker/frontend/src/engine/sensorWhite.ts b/docker/frontend/src/engine/sensorWhite.ts new file mode 100644 index 0000000..af52574 --- /dev/null +++ b/docker/frontend/src/engine/sensorWhite.ts @@ -0,0 +1,54 @@ +// Where this frame's sensor actually ran out, in the plane's own counts. +// +// `maximum` is the format's full scale, not the sensor's: a frame that has blown +// plateaus at 1.89-2.00x it (measured, as ratios of `maximum - black`: Sony +// ILCE-5100 2.002, Ricoh GR III 1.964, Fuji X-T3 / X100V / X100S 1.892) and the +// white level the shader divides by has to be that plateau, or the frame lands +// short of white and the highlight desaturation — a smoothstep that starts at +// 0.95 of the *sensor* — never fires on a body that sits at 0.946. Reading it +// off the frame is what makes one develop right for every body. +// +// A plateau, not a max: the frame is read over the nine counts at its top, and +// the white level is the highest of those that is a **cliff** — a count the one +// below it does not come near. A clipped region piles at the level the sensor +// stops at, so the count under that level is the sensor's own noise floor and +// orders of magnitude smaller; a smooth bright sky hands its top count to the +// next one down in similar numbers and is left alone. A frame that has not +// clipped — and one that never reaches the format's own scale — stays on the old +// factor two, the population mean of the ratios above and the best guess when +// there is nothing to measure. +// +// One count down, not the largest of the three: a sensor clips its channels at +// counts a couple apart, so the plane carries a second pile just under the top +// one, and asking the top pile to beat *that* pile refused every frame whose +// channels do not run out together. Measured on a Fujifilm XF10 RAF: 433819 +// samples at 30993 over 30 at 30992 — a cliff — and 83954 at 30990 three counts +// under it, which the old rule read as the ground the top pile had to clear eight +// times over (it clears 44, not 83954) and answered "not clipped" for a frame +// holding 6.7% of its plane at the level. Read that way the develop divided by +// 2.000x instead of 1.953x, its plateau landed on 0.976 of the sensor, the +// desaturation fired at half strength, and the highlights kept the magenta the +// white balance gains make of them (36.7% of the frame's bright blocks red and +// blue of green against 0.0% in the camera's own JPEG of the same shot). +// +// The pile is still asked to be a few dozen samples and not a share of the plane: +// what a blown region is has nothing to do with how many the plane holds, and the +// frame that has blown one lamp is the frame whose highlights need the level most. +export function sensorWhite(data: ArrayLike, maximum: number, black: number): number { + const legacy = 2 * (maximum - black); + const n = data.length; + let top = 0; + for (let i = 0; i < n; i++) if (data[i] > top) top = data[i]; + if (top < maximum - black) return legacy; + const lo = top - 8; + const near = new Uint32Array(9); + for (let i = 0; i < n; i++) { const v = data[i]; if (v >= lo) near[v - lo]++; } + const floor = Math.max(32, Math.ceil(n * 1e-6)); + // The top count can be a stray sample above the sensor's level; the pile under + // it is the level itself, so the scan starts at the top and takes the first + // count that is both a pile and a cliff over the count below it. + for (let k = 8; k >= 1; k--) { + if (near[k] >= floor && near[k] >= near[k - 1] * 8) return lo + k; + } + return legacy; +}