// RAW → JPEG on the client, so a camera's own file opens in the studio without // a DNG converter in the middle (see native_raw_processing_opfs_architecture.md). // // A RAW opens at the colour the camera chose for it. LibRaw is kept out of white // balance and tone (below) and knows nothing of the body's picture style, so the // develop is fitted to the one rendering the file does carry with it: the preview // the camera wrote inside it, which is the frame the photographer saw and the one // a desktop viewer shows. Develop the sensor, take a block grid out of each on // the encoded values, fit the cubic a channel between them, and develop the // sensor through that curve (see previewMatch.ts). // // The curve is the whole fit. A 3x3 was least squared ahead of it for a long // time and is what put the olive cast on the Olympus frame this was reported on: // a develop keeps its luminance and its colour in the same three numbers and the // luminance is two orders of magnitude the larger, so a least squares dominated // by it holds the channel means and buys them with the colour axis. The camera's // picture style is a curve, not a matrix. // // Measured on that frame, standard deviation of R-G over it (the file's own // preview 8.3, the camera's own JPEG 12.3, an olive frame being one where this // has gone to nothing): 11.6 as the develop left it, 11.9 through the curve // alone, 4.6 through the curve under the fitted 3x3. What no per-pixel transform // reaches either way is the camera's own sharpening. // // Developing the sensor is also what keeps the highlights: 2.3% of the frame at // pure white against 3.3% in the preview itself, since the 8-bit preview threw // the headroom away and the develop still has it. The preview is therefore the // reference and the fallback, not the frame: it is what a file with no usable // preview (some DNG), or with sensor data that will not decode, opens as. // // The develop: LibRaw demosaics in its own worker; what comes back is linear camera data, // which this file turns into the sRGB the rest of the pipeline expects. The // settings below were checked against the preview a Sony ILME-FX30 writes into // its own ARW (the camera JPEG, read straight out of the file): the developed // frame and that preview agree to within 1% on both channel ratios, R/G 0.909 // against 0.903 and B/G 0.603 against 0.606. // - noAutoScale + useCameraWb:false + noAutoBright + gamm [1,1] keep LibRaw out // of white balance and tone, so `cam_mul` and `rgb_cam` can be applied here // exactly once. // - halfSize:false: the frame opens at the sensor's own resolution — a 24MP // RAW develops to 24MP (6020x4024 on the GR, 6000x4000 on the Fuji), not to // the quarter the half-size demosaic reports. `userQual` 3 then demosaics // all of it. The develop was half-size until a GR's DNG opened at 3010x2012 // against its own JPEG's 6000x4000: the quarter-size frame was the flag, // not the file. ponytail: costs ~4x the develop time and two full-size F16 // surfaces; put the flag behind a "draft" toggle if a phone ever has to. // // The band loop exists because a single Float32 copy of the whole plane would be // ~100MB. Each band is decoded, normalised and drawn before the next is read. // ponytail: the develop itself is one full-resolution shader draw per band, on // the main thread (Skia is not available in the RAW worker). Move it to a worker // with an OffscreenCanvas if the develop ever blocks the UI visibly. import LibRaw from 'libraw-wasm'; import { sensorWhite } from './sensorWhite'; import { f32ToF16 } from './halfFloat'; import { toneMatch, FLAT_TONE, MATCH_GRID } from './previewMatch'; import { Skia, initSkia } from './skiaShim'; import { decodeTiff } from './tiffDecode'; // What `imageData()` returns for the settings below: 16-bit, 3 channels, black // level already gone — the post-process subtracts it whatever `noAutoScale` // says, which only holds back the white balance and the output scaling. // // Its white level is not `maximum` but the frame's own plateau, a factor of 1.89 // to 2.00 out (see sensorWhite below). The gain carries that level, so the white // lands back on 1.0 — and on every body, not just the two that factor two was // fitted on: without it every frame opened a stop bright (the FX30's own JPEG has // 0.05% of pixels at pure white where the develop had 0.76%) and a body that sat // lower (X-Trans, 1.892) never even reached the highlight desaturation, which // starts at 0.95 of the sensor. const SETTINGS = { halfSize: false, outputBps: 16, outputColor: 0, noAutoScale: true, useCameraWb: false, noAutoBright: true, gamm: [1, 1] as [number, number], userQual: 3, highlight: 2, }; // 16-bit samples, but everything below is [0,1] — Skia reads an F32 image as // already-normalised colour, and a raw 0..65535 plane comes back black. const SAMPLE_MAX = 65535; // Pixels per band, held under the point where the Float32 copy dominates the // memory the page is allowed to use. const BAND_PIXELS = 2_000_000; const RAW_DEVELOP_SKSL = ` uniform shader raw; uniform float gain; // 1 / the white level the frame itself ran out at uniform float4 mul; // cam_mul, green-normalised uniform float4 m0; // camera -> sRGB, the first three columns of rgb_cam uniform float4 m1; uniform float4 m2; uniform float4 crop; // (y offset of this band, 0, 0, 0) uniform float4 t0; // the file's own rendering, one cubic a channel on the uniform float4 t1; // encoded value: r a b c d per float4 (flat when there is uniform float4 t2; // no preview to fit to) // The camera's rendering is a shape, and this is that shape, read on the value // the develop left. float tone(float4 w, float x) { return clamp(w.x + x * (w.y + x * (w.z + x * w.w)), 0.0, 1.0); } float3 encode(float3 x) { x = clamp(x, 0.0, 1.0); return mix(x * 12.92, 1.055 * pow(x, float3(1.0 / 2.4)) - 0.055, step(float3(0.0031308), x)); } half4 main(float2 pos) { float4 p = raw.eval(float2(pos.x, pos.y - crop.x)); // The plane arrives with the black level already subtracted — it floors at 0, // not at color_data.black (measured on the FX30 ARW: the sensor mosaic floors // at 334, the plane at 0, a quarter of its red samples under the black level). // Subtracting it again drained red and blue — the two channels the gains lift // most — and dragged every frame towards green. float3 n = max(p.rgb * gain, 0.0); // the sensor's own levels, white level 1.0 float3 lin = n * mul.rgb; // Only the floor there. A photo the sensor could not hold goes over the white // level in all three channels, and clipping them one by one before the WB gains // is what tints what is left of the highlight: green — the channel the gains are // normalised to — stops at 1.0 while red and blue, which need their 2.6x and // 1.6x, are already past it, so the blown area comes out magenta. Keep the // channel ratios through the matrix instead and let the overflow fade to white. float3 rgb = float3(dot(m0.xyz, lin), dot(m1.xyz, lin), dot(m2.xyz, lin)); float mx = max(max(rgb.r, rgb.g), rgb.b); // A pixel that has run to the white level has no colour of its own left to keep, // and what the gains made of it is an artefact, not a colour: ease the pixel // towards the neutral of its own value as that point is approached. The clip to // read is the one the gains make, not only the sensor's own — the gains here are // 1.7x and 1.9x on red and blue, so a blown sky reaches the white level at 0.59 of // the raw range in those channels while the green, which the gains are // normalised to, only reaches it at 1.0. Read on the sensor's levels alone the // gate stayed shut across a whole blown sky and left the develop's magenta in it // (227,184,245 at the gate's own value against 245,245,245 read where the gains // put the clip, the camera's preview white at that block). Both are read, so a // body whose gains do not lift a channel keeps the sensor's own clip as its gate. // mx is the value to hold: it is the pixel's own lightness. float hi = max(max(n.r, n.g), n.b); hi = max(hi, max(max(lin.r, lin.g), lin.b)); float blown = smoothstep(0.82, 0.98, hi); rgb = mix(rgb, float3(mx), blown); // The overflow used to fade towards white — mix(rgb / mx, 1, 1 - 1 / mx) — // which put every pixel of a blown sky on exactly 1.0 and threw the two stops // the sensor held above white away with it: LIGHT's HIGHLIGHT row then had a // flat white to pull on and nothing to reveal. The white point is moved down // instead and the overflow squeezed back in under it by the doc's soft knee: // // y = T + over / (1 + 2S*over), over = mx - T, S = 1 / (2(1 - T)) // // S is what puts 1.0 on the asymptote, so the sensor's own plateau — two white // levels up, see the gain above — lands at ~0.94 and the first stop over white // spends 0.85..0.94. Below T the frame is untouched and the curve leaves T with // the slope it arrived with (1), so there is no seam to mask; above it the frame // darkens, which is the one move no later pass can undo — which is the point, // and what the per-file curve below then measures the REST of the frame back from. // // ponytail: 4.7 stops of headroom now share ~5% of the ramp, and the develop // still leaves as an 8-bit JPEG. Give it a float16 output when RAW highlights // have to print rather than merely be seen. if (mx > 0.7) { float over = mx - 0.7; rgb *= (0.7 + over / (1.0 + over * 3.3333)) / mx; } float3 e = encode(rgb); // The file's own colour: the curve, in float, on the encoded value it was fitted // on — and the last step the frame leaves through. Not the 8-bit colour filter // this used to be painted through: the curve carries an exposure (the preview is // the reference, so matching its brightness is part of matching its colour), and // it is what carries the body's own tone — a 3x3 can only scale, and the one that // used to sit here bought the channel means by collapsing the colour axis it was // fitted against (see previewMatch.ts). No rolloff: white is one of the values // the curve is fitted on, and it is pinned there, so a blown pixel still lands on // white while the frame's own highlights stay where the knee above left them. float3 o = float3(tone(t0, e.r), tone(t1, e.g), tone(t2, e.b)); // ...and the neutral the gate above drew is what the curve undoes: one cubic a // channel, fitted on a grid that has no block left to fit where the frame ran out // (the blown blocks are excluded, see previewMatch), so at the plateau the three // curves agree only at 1.0 and part company either side of it — measured on the // ORF, a 255,255,255 sky came back 254,255,255 and 253,255,254, red under green // across a quarter of the frame. The gate is the develop's own statement that the // pixel had no colour of its own, so it is re-read here, on the value that leaves. return half4(half3(mix(o, float3(max(max(o.r, o.g), o.b)), blown)), 1.0); } `; // The extensions LibRaw is compiled for. Not `tif`: a TIFF in a camera folder is // as likely to be an export as a RAW, and opening one asked for PRO and wore a // RAW badge for a file the engine has nothing to develop in. It is read for its // own pixels instead — see `tiffDevelop`. const RAW_EXT = [ 'arw', 'srf', 'sr2', 'cr2', 'cr3', 'crw', 'nef', 'nrw', 'dng', 'orf', 'raf', 'rw2', 'pef', 'srw', 'raw', 'rwl', '3fr', 'fff', 'iiq', 'mos', 'mrw', 'x3f', ]; export const RAW_ACCEPT = RAW_EXT.map((e) => `.${e}`).join(','); export function isRawName(name: string): boolean { const ext = name.slice(name.lastIndexOf('.') + 1).toLowerCase(); return name.includes('.') && RAW_EXT.includes(ext); } // A TIFF is not a RAW: it is a picture no decoder here draws by itself, and the // one test for it, so the tile's route and the studio's route cannot drift apart. export function isTiffName(name: string): boolean { return /\.(tiff?)$/i.test(name); } // The camera's own preview, when the file carries one: the colour reference the // curve below is fitted to, and what the file opens as when the sensor does not // decode. // ponytail: it is 1616x1080 on an A5100 and 1620x1080 on an FX30, so handing it // back is a 1.7MP frame — a print past it has to come off the develop, which is // what a developed file already gives. async function cameraPreview(raw: LibRaw): Promise { const thumb = await raw.thumbnailData().catch(() => undefined); if (thumb?.format !== 'jpeg' || !thumb.data?.length) return null; return new Uint8Array(thumb.data); } // Fast pure JS binary scanner to extract embedded JPEG preview from RAW files (ARW, NEF, CR2, CR3, DNG, RAF, RW2, ORF, PEF). // Scans for all embedded JPEGs in the RAW file and returns the largest segment (the full-size camera preview). export function extractEmbeddedJpeg(buf: Uint8Array): Uint8Array | null { const len = buf.length; let largest: Uint8Array | null = null; let maxLen = 0; for (let i = 0; i < len - 4; i++) { if (buf[i] === 0xff && buf[i + 1] === 0xd8 && buf[i + 2] === 0xff) { const start = i; let pos = i + 2; let eoiPos = -1; let validJpeg = true; while (pos < len - 4) { if (buf[pos] !== 0xff) { validJpeg = false; break; } const m = buf[pos + 1]; if (m === 0xd9) { eoiPos = pos + 2; break; } if (m === 0xda) { const sosHeaderLen = (buf[pos + 2] << 8) | buf[pos + 3]; if (sosHeaderLen < 2 || pos + 2 + sosHeaderLen > len) { validJpeg = false; break; } let scanPos = pos + 2 + sosHeaderLen; const searchEnd = Math.min(len - 1, scanPos + 20 * 1024 * 1024); while (scanPos < searchEnd) { if (buf[scanPos] === 0xff) { const marker = buf[scanPos + 1]; if (marker === 0xd9) { eoiPos = scanPos + 2; break; } if (marker === 0x00 || (marker >= 0xd0 && marker <= 0xd7)) { scanPos += 2; continue; } } scanPos++; } break; } if (m === 0x00 || (m >= 0xd0 && m <= 0xd7)) { pos += 2; continue; } const segLen = (buf[pos + 2] << 8) | buf[pos + 3]; if (segLen < 2 || pos + 2 + segLen > len) { validJpeg = false; break; } pos += 2 + segLen; } if (eoiPos > start && validJpeg) { const segSize = eoiPos - start; if (segSize > 10000 && segSize > maxLen) { maxLen = segSize; largest = buf.subarray(start, eoiPos); } i = eoiPos - 1; } } } return largest; } // The preview on its own, for the catalogue: a folder of RAW files has to show // a tile per frame, and unpack_thumb is a seek and a copy where the develop // above is a full decode of every pixel at full resolution. No preview inside // the file means no tile — the row still lists the frame by name, and the // studio develops it the moment it is opened. // // What CanvasKit draws, and only that: a TIFF goes to `decodeTiff` below, since // `MakeImageFromEncoded` answers null to one (measured on a baseline 5472x3648 // scan) and the file has to be read by hand. function tiffImage(bytes: Uint8Array, maxSide: number): any { const img = Skia.Image.MakeImageFromEncoded(bytes); if (img) return img; const frame = decodeTiff(bytes, maxSide); if (!frame) return null; return Skia.Image.MakeImage( { width: frame.width, height: frame.height, colorType: Skia.ColorType.RGBA_8888, alphaType: Skia.AlphaType.Unpremul, colorSpace: Skia.ColorSpace.SRGB, }, frame.rgba, frame.width * 4 ); } // The same read, encoded to JPEG. `max` is the long side it comes out at, 0 for // the file's own size. function tiffJpeg(bytes: Uint8Array, max: number): Uint8Array | null { try { const img = tiffImage(bytes, max || Infinity); if (!img) return null; try { return shrinkToJpeg(img, max || Math.max(img.width(), img.height())); } finally { img.delete(); } } catch { return null; } } export function tiffThumbnail(bytes: Uint8Array): Uint8Array | null { return tiffJpeg(bytes, 640); } // A `.tif` read for the screen, now that it is no file the engine develops and // no decoder the browser has: the frame is the picture, so it is decoded here — // by the studio at the file's own size, by the catalogue's stage at what a screen // shows (`max`, 0 for the file's own). CanvasKit is what draws it, and neither // caller has booted it: a visitor who only ever opens the wall never does. // // ponytail: at the file's own size this is the frame's RGBA in memory (80MB for a // baseline 5472x3648 scan) plus the canvas it is drawn to. Cap it if a plate ever // arrives that does not fit; the develop above is the same order already. export async function tiffDevelop(bytes: Uint8Array, max = 0): Promise { if (!Skia) await initSkia({ locateFile: (file) => `/wasm/${file}` }); return tiffJpeg(bytes, max); } // A frame shrunk to the square the wall paints and encoded to JPEG — the one // step of a tile that is CanvasKit's whatever the frame came out of. function shrinkToJpeg(img: any, n = 640): Uint8Array | null { const w = img.width(); const h = img.height(); if (!w || !h) return null; const wide = w >= h; const targetW = wide ? n : Math.max(1, Math.round((w / h) * n)); const targetH = wide ? Math.max(1, Math.round((h / w) * n)) : n; const surface = Skia.Surface.MakeOffscreen(targetW, targetH) ?? Skia.Surface.Make(targetW, targetH); if (!surface) return null; const canvas = surface.getCanvas(); canvas.drawImageRectCubic( img, Skia.XYWHRect(0, 0, w, h), Skia.XYWHRect(0, 0, targetW, targetH), 1 / 3, 1 / 3 ); surface.flush(); const snapshot = surface.makeImageSnapshot(); surface.dispose(); if (!snapshot) return null; const jpeg = snapshot.encodeToBytes(Skia.ImageFormat.JPEG, 80); snapshot.dispose(); return jpeg ? new Uint8Array(jpeg) : null; } // The preview on its own, for the catalogue: a folder of RAW files has to show // a tile per frame, and unpack_thumb is a seek and a copy where the develop // above is a full decode of every pixel at full resolution. No preview inside // the file means no tile — the row still lists the frame by name, and the // studio develops it the moment it is opened. export async function rawThumbnail(bytes: Uint8Array, fileName?: string): Promise { // A `.tif` is not a RAW: the file IS the picture, and no camera preview is // hiding in it — scanning the bytes for one finds the frame's own pixel data // and hands back three hundred kilobytes of it dressed as a JPEG, which is // what the wall used to keep as the frame's tile and draw as nothing. if (fileName && isTiffName(fileName)) { // CanvasKit is what draws the JPEG inside one, and the catalogue has a route // of its own: the studio boots the engine, a visitor who only ever opens the // wall never does, and a tile read before it is up draws as nothing. if (!Skia) await initSkia({ locateFile: (file) => `/wasm/${file}` }); return tiffThumbnail(bytes); } const fast = extractEmbeddedJpeg(bytes); if (fast) return fast; const raw = new LibRaw(); try { // The same transfer as in the develop: the worker is handed a copy, so the // caller's bytes stay whole for whatever the catalogue reads off them next. await raw.open(bytes.slice() as unknown as BufferSource, SETTINGS); return await cameraPreview(raw); } catch { return tiffThumbnail(bytes); } finally { try { raw.dispose(); } catch {} } } // MATCH_GRID x MATCH_GRID block colours of a frame, one byte per channel. function gridOf(image: any, n = MATCH_GRID): Uint8Array | null { const surface = Skia.Surface.MakeOffscreen(n, n) ?? Skia.Surface.Make(n, n); if (!surface) return null; const canvas = surface.getCanvas(); // Cubic, not a linear tap: this is a 45x reduction and linear reads a handful // of source pixels per block — noise for the curve to be fitted on. canvas.drawImageRectCubic( image, Skia.XYWHRect(0, 0, image.width(), image.height()), Skia.XYWHRect(0, 0, n, n), 1 / 3, 1 / 3 ); surface.flush(); const px = canvas.readPixels(0, 0, { width: n, height: n, colorType: Skia.ColorType.RGBA_8888, alphaType: Skia.AlphaType.Unpremul, colorSpace: Skia.ColorSpace.SRGB, }) as Uint8Array | null; surface.dispose(); return px ? new Uint8Array(px.buffer, px.byteOffset, px.byteLength) : null; } // The same grid out of the preview, which the file carries as a JPEG. Decoded and // reduced through the same Skia call as the develop's own grid, because the two // grids are only comparable — and a curve between them only meaningful — when one // resampler made both. A 2D canvas here instead left the curve following its own // smoothing: on the A5100 frame the same develop scored dE00 4.7 against 4.4, and // the dark end of the frame came out 6 L further from the preview than the curve // it was handed asked for. function previewGrid(jpeg: Uint8Array, w: number, h: number, n = MATCH_GRID): Uint8Array | null { const bmp = Skia.Image.MakeImageFromEncoded(jpeg); if (!bmp) return null; try { if (bmp.width() < n || bmp.height() < n) return null; // The decode above has already turned the preview by its own EXIF // orientation: a 1920x1280 orientation-8 Panasonic preview decodes to // 1280x1920, the frame the camera meant. So these are display dimensions and // the only thing left is to trim them to the develop's aspect. Rotating here // as well — which this did — turned a portrait frame's reference a quarter // turn against the develop it was being fitted to, and the fit was rejected // as folded (see previewMatch), which is why the Panasonic RW2 opened with no // curve at all. const targetAspect = w / h; const bmpAspect = bmp.width() / bmp.height(); let cropW = bmp.width(); let cropH = bmp.height(); if (Math.abs(bmpAspect - targetAspect) > 0.01) { if (bmpAspect > targetAspect) { cropW = Math.round(bmp.height() * targetAspect); } else { cropH = Math.round(bmp.width() / targetAspect); } } const cropX = Math.round((bmp.width() - cropW) / 2); const cropY = Math.round((bmp.height() - cropH) / 2); const surface = Skia.Surface.MakeOffscreen(n, n) ?? Skia.Surface.Make(n, n); if (!surface) return null; const canvas = surface.getCanvas(); canvas.drawImageRectCubic( bmp, Skia.XYWHRect(cropX, cropY, cropW, cropH), Skia.XYWHRect(0, 0, n, n), 1 / 3, 1 / 3 ); surface.flush(); const px = canvas.readPixels(0, 0, { width: n, height: n, colorType: Skia.ColorType.RGBA_8888, alphaType: Skia.AlphaType.Unpremul, colorSpace: Skia.ColorSpace.SRGB, }) as Uint8Array | null; surface.dispose(); return px ? new Uint8Array(px.buffer, px.byteOffset, px.byteLength) : null; } finally { bmp.delete(); } } // The camera -> sRGB matrix, applied to the WB'd sensor triple. // // `rgb_cam` is dcraw's own: it is the camera -> sRGB transform already, its rows // sum to one, and the 1/num dcraw scales it by rides in `pre_mul`. Dividing it by // pre_mul a second time undoes the normalisation and drags the frame towards // green. It is what every body LibRaw knows a `cam_xyz` for gets — the Olympus // ORF this was reported on included, whose `rgb_cam` and `cam_xyz` disagree // (measured: rgb_cam x cam_mul lands on R/G 1.021 B/G 0.920 against the file's own // preview's 1.013 / 0.841, the cam_xyz chain this used to prefer on 0.921 / 0.886 // — green, and blue for a scene the camera left neutral). // // `cam_xyz` stays as the fallback for a file that has one and no `rgb_cam`: its // rows are the XYZ of each camera channel, so sRGB = xyz_rgb^-1 . cam_xyz^T . cam, // with the rows normalised so a neutral camera triple opens neutral, as dcraw's // cam_xyz_coeff normalises them. const XYZ_TO_SRGB = [ [3.2404542, -1.5371385, -0.4985314], [-0.9692660, 1.8760108, 0.0415560], [0.0556434, -0.2040259, 1.0572252], ]; export type CameraBrand = | 'fujifilm' | 'sony' | 'canon' | 'nikon' | 'olympus' | 'panasonic' | 'leica' | 'apple' | 'ricoh' | 'dng' | 'generic'; export function detectCameraBrand(fileName?: string, meta?: any): CameraBrand { const ext = fileName ? fileName.slice(fileName.lastIndexOf('.') + 1).toLowerCase() : ''; const make = (meta?.idata?.make || meta?.make || '').toLowerCase(); if (ext === 'raf' || make.includes('fuji')) return 'fujifilm'; if (ext === 'arw' || ext === 'srf' || ext === 'sr2' || make.includes('sony')) return 'sony'; if (ext === 'cr2' || ext === 'cr3' || ext === 'crw' || make.includes('canon')) return 'canon'; if (ext === 'nef' || ext === 'nrw' || make.includes('nikon')) return 'nikon'; if (ext === 'orf' || make.includes('olympus') || make.includes('om digital')) return 'olympus'; if (ext === 'rw2' || make.includes('panasonic') || make.includes('lumix')) return 'panasonic'; if (ext === 'pef' || make.includes('ricoh') || make.includes('pentax')) return 'ricoh'; if (make.includes('leica')) return 'leica'; if (make.includes('apple') || make.includes('iphone')) return 'apple'; if (ext === 'dng') return 'dng'; return 'generic'; } export function getBrandWbMultipliers(brand: CameraBrand, cd: any): number[] { // 1. The balance the body recorded with the frame. `pre_mul` is the daylight one // LibRaw carries for the model, and it is what used to be read first — on the // Fujifilm RAF this was reported on it is the wrong one twice over (R/G 2.296 // against the frame's own 1.795, B/G 1.307 against 1.626), which left blue short // and every white in the frame pink. Measured against the file's own preview over // the sample set, dE00 6.73 -> 5.27 on that frame and 7.52 -> 7.26 on the X-T20's. if (cd?.cam_mul && Array.isArray(cd.cam_mul) && cd.cam_mul.some((v: number) => v > 0 && v !== 1)) { return cd.cam_mul; } // 2. Otherwise the daylight balance LibRaw worked out for the model. if (cd?.pre_mul && Array.isArray(cd.pre_mul) && cd.pre_mul.some((v: number) => v > 0 && v !== 1)) { return cd.pre_mul; } // 3. Brand-calibrated WB fallbacks (prevents generic single fallback from producing color casts) switch (brand) { case 'fujifilm': return [2.24, 1.0, 1.72, 1.0]; case 'sony': return [1.95, 1.0, 1.55, 1.0]; case 'canon': return [2.10, 1.0, 1.48, 1.0]; case 'nikon': return [1.88, 1.0, 1.42, 1.0]; case 'olympus': return [2.15, 1.0, 1.38, 1.0]; case 'panasonic': return [1.98, 1.0, 1.45, 1.0]; case 'leica': return [1.85, 1.0, 1.65, 1.0]; case 'apple': return [2.05, 1.0, 1.82, 1.0]; case 'ricoh': return [1.96, 1.0, 1.52, 1.0]; case 'dng': default: return [2.00, 1.0, 1.50, 1.0]; } } // The balance the file's own preview was rendered with. // // Neither balance LibRaw hands over is it. `cam_mul` is what the body recorded and // `pre_mul` the daylight one it carries for the model, and the preview in the file // is the rendering the body actually made — so the balance is in there, read off the // blocks the camera left bright and near neutral: average the sensor triple behind // each of those blocks over the block's own pixels, and the triple that makes those // blocks come out equal is the frame's white. The camera's rendering is the same // reference the tone curve below is fitted to; this is its WB row. // // Read on the frame, not on the file's tags, which is why it holds across bodies: // measured against the embedded preview, the Fujifilm RAF this was reported on came // back at block a* -1.9 against the preview's -2.1 where `pre_mul` gave 0.0 and // `cam_mul` -1.0 — the pink in every white of the frame — and dE00 4.93 where they // gave 6.73 and 5.27. On the X-T20 frame: -4.2 against -3.9, dE00 6.56 against 7.52 // and 7.26. Both notes were magenta before either balance was tried. // // Blocks and not samples: a block is a few thousand sensor pixels and its mean is // what the tone curve is fitted on, so the two read the same frame. A clipped block // is skipped on both sides — the camera's rendering has no ratio left in it and // neither has the sensor — and a frame that offers too few blocks for a balance // keeps the tags above rather than a figure read off noise. export function previewWhiteBalance( data: ArrayLike, w: number, h: number, grid: Uint8Array, n: number, white: number ): number[] | null { const sums = [0, 0, 0]; let count = 0; const clip = white * 0.95; for (let by = 0; by < n; by++) { for (let bx = 0; bx < n; bx++) { const i = (by * n + bx) * 4; const r = grid[i]; const g = grid[i + 1]; const b = grid[i + 2]; const hi = Math.max(r, g, b); const lo = Math.min(r, g, b); if (lo < 160 || hi - lo > 24 || hi > 250) continue; const x0 = Math.floor((bx * w) / n); const x1 = Math.max(x0 + 1, Math.floor(((bx + 1) * w) / n)); const y0 = Math.floor((by * h) / n); const y1 = Math.max(y0 + 1, Math.floor(((by + 1) * h) / n)); for (let y = y0; y < y1; y++) { let o = (y * w + x0) * 3; for (let x = x0; x < x1; x++, o += 3) { const pr = data[o]; const pg = data[o + 1]; const pb = data[o + 2]; if (pr > clip || pg > clip || pb > clip) continue; sums[0] += pr; sums[1] += pg; sums[2] += pb; count++; } } } } if (count < 4096 || sums[0] <= 0 || sums[1] <= 0 || sums[2] <= 0) return null; const mul = [sums[1] / sums[0], 1, sums[1] / sums[2], 1]; return mul.every((v) => v > 0.4 && v < 3) ? mul : null; } export function getCamToSrgbMatrix(cd: any, brand?: CameraBrand): number[][] { if (cd?.rgb_cam && Array.isArray(cd.rgb_cam) && cd.rgb_cam.length >= 3) { return [0, 1, 2].map((i) => [0, 1, 2].map((j) => cd.rgb_cam[i]?.[j] || 0)); } if (cd?.cam_xyz && Array.isArray(cd.cam_xyz) && cd.cam_xyz.length >= 3) { const cx = cd.cam_xyz; const scale = (Math.abs(cx[0][0]) > 100) ? 10000.0 : 1.0; const m = [0, 1, 2].map((i) => [0, 1, 2].map((j) => XYZ_TO_SRGB[i][0] * ((cx[j]?.[0] || 0) / scale) + XYZ_TO_SRGB[i][1] * ((cx[j]?.[1] || 0) / scale) + XYZ_TO_SRGB[i][2] * ((cx[j]?.[2] || 0) / scale) ) ); return m.map((row) => { const num = row[0] + row[1] + row[2]; return Math.abs(num) < 1e-6 ? row : row.map((v) => v / num); }); } // Brand-calibrated default matrix when rgb_cam and cam_xyz are absent switch (brand) { case 'fujifilm': return [[1.15, -0.08, -0.07], [-0.15, 1.25, -0.10], [-0.05, -0.20, 1.25]]; case 'sony': return [[1.18, -0.12, -0.06], [-0.12, 1.22, -0.10], [-0.04, -0.18, 1.22]]; case 'canon': return [[1.12, -0.06, -0.06], [-0.10, 1.20, -0.10], [-0.04, -0.16, 1.20]]; case 'nikon': return [[1.14, -0.08, -0.06], [-0.11, 1.21, -0.10], [-0.04, -0.17, 1.21]]; default: return [[1.0, 0, 0], [0, 1.0, 0], [0, 0, 1.0]]; } } // The size a JPEG declares for itself, off its SOF segment — a header read, no // decoder. Null when the bytes carry no readable frame header. export function jpegFrameSize(jpeg: Uint8Array): { width: number; height: number } | null { for (let i = 2; i + 9 < jpeg.length; ) { if (jpeg[i] !== 0xff) { i++; continue; } const marker = jpeg[i + 1]; if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) { return { height: (jpeg[i + 5] << 8) | jpeg[i + 6], width: (jpeg[i + 7] << 8) | jpeg[i + 8] }; } i += 2 + ((jpeg[i + 2] << 8) | jpeg[i + 3]); } return null; } // Is the embedded preview the frame itself? Bodies that write one at the frame // size (Nikon NEF 6016x4016 out of 6032x4032, Sony ARW, an iPhone ProRAW DNG) // differ from the sensor by a margin row or two; a thumbnail out of the same // sensor is 1920x1280, a third of the long side, so 90% divides them cleanly. // Unreadable sizes keep the old behaviour and trust the preview. function previewIsFrame(preview: Uint8Array, meta: any): boolean { const size = jpegFrameSize(preview); const frame = Math.max( Number(meta?.iwidth) || 0, Number(meta?.iheight) || 0, Number(meta?.width) || 0, Number(meta?.height) || 0, Number(meta?.raw_width) || 0, Number(meta?.raw_height) || 0 ); if (!size || !frame) return false; return Math.max(size.width, size.height) >= frame * 0.9; } export async function developRaw(bytes: Uint8Array, fileName?: string): Promise { // The embedded JPEG is the camera's own rendering — exact colour, tone, white // balance and picture profile, generated by the body itself. It is the frame // only when it is the size of the frame; on most bodies it is a thumbnail // (1920x1280 out of a 24MP sensor, measured on a Fuji RAF and a Lumix RW2), so // handing it back is what opened a 24MP file as a 1920 wide one. Whether it is // the frame is known from the sensor size, which costs one LibRaw open (91ms // measured, metadata only — no demosaic) against seconds for the develop. const preview = extractEmbeddedJpeg(bytes); const raw = new LibRaw(); try { await raw.open(bytes.slice() as unknown as BufferSource, SETTINGS); const meta = await raw.metadata(true); if (preview && preview.length > 10000 && previewIsFrame(preview, meta)) { return preview; } const img = await raw.imageData(); const cd = meta?.color_data; if (!img || (!cd?.cam_mul && !cd?.pre_mul) || (!cd.rgb_cam && !cd.cam_xyz)) { const camPreview = await cameraPreview(raw); if (camPreview && camPreview.length > 10000) return camPreview; throw new Error('RAW has no colour data'); } const { width: w, height: h } = img; const data = img.data as Uint16Array; if (!w || !h) throw new Error('RAW decoded to nothing'); const effect = Skia.RuntimeEffect.Make(RAW_DEVELOP_SKSL); if (!effect) throw new Error('develop shader failed to compile'); const brand = detectCameraBrand(fileName, meta); const white = sensorWhite(data, cd.maximum, cd.black); // Read before the develop, not after: the balance and the curve are both read // off the file's own preview, and the preview map costs one decode either way. const refGrid = preview ? previewGrid(preview, w, h, MATCH_GRID) : null; const rawMul = (refGrid && previewWhiteBalance(data, w, h, refGrid, MATCH_GRID, white)) ?? getBrandWbMultipliers(brand, cd); const green = rawMul[1] || rawMul[3] || 1; const mul = rawMul.map((v: number) => v / green); const [r0, r1, r2] = getCamToSrgbMatrix(cd, brand); const bandH = Math.max(1, Math.min(h, Math.floor(BAND_PIXELS / w))); const f32 = new Float32Array(w * bandH * 4); const half = new Uint16Array(w * bandH * 4); const uniforms = new Float32Array(33); uniforms[0] = SAMPLE_MAX / white; uniforms.set([mul[0], mul[1], mul[2], 0, r0[0], r0[1], r0[2], 0, r1[0], r1[1], r1[2], 0, r2[0], r2[1], r2[2], 0], 1); const develop = (tone: Float32Array | null) => { const surface = Skia.Surface.MakeOffscreen(w, h) ?? Skia.Surface.Make(w, h); if (!surface) return null; uniforms.set(tone ?? FLAT_TONE, 21); for (let y0 = 0; y0 < h; y0 += bandH) { const rows = Math.min(bandH, h - y0); let o = 0; for (let i = y0 * w * 3, end = (y0 + rows) * w * 3; i < end; i += 3) { f32[o++] = data[i] / SAMPLE_MAX; f32[o++] = data[i + 1] / SAMPLE_MAX; f32[o++] = data[i + 2] / SAMPLE_MAX; f32[o++] = 1; } f32ToF16(f32, half, w * rows * 4); const band = Skia.Image.MakeImage( { width: w, height: rows, colorType: Skia.ColorType.RGBA_F16, alphaType: Skia.AlphaType.Unpremul }, new Uint8Array(half.buffer, 0, w * rows * 8), w * 8 ); if (!band) throw new Error('band image failed'); const child = band.makeShaderOptions( Skia.TileMode.Clamp, Skia.TileMode.Clamp, Skia.FilterMode.Nearest, Skia.MipmapMode.None ); uniforms[17] = y0; const shader = effect.makeShaderWithChildren(uniforms, [child]); const paint = Skia.Paint(); paint.setShader(shader); surface.getCanvas().drawRect(Skia.XYWHRect(0, y0, w, rows), paint); surface.flush(); paint.delete(); shader.delete(); child.delete(); band.delete(); } const shot = surface.makeImageSnapshot(); surface.dispose(); return shot; }; const first = develop(null); if (!first) throw new Error('no surface for the develop'); let finalShot = first; let refJpeg = preview; if (!refJpeg) { refJpeg = await cameraPreview(raw).catch(() => null); } if (refJpeg) { try { const devGrid = gridOf(first, MATCH_GRID); const matchGrid = refGrid ?? previewGrid(refJpeg, w, h, MATCH_GRID); if (devGrid && matchGrid) { const fittedTone = toneMatch(devGrid, matchGrid, MATCH_GRID); if (fittedTone) { const second = develop(fittedTone); if (second) { finalShot = second; first.dispose(); } } } } catch { // Keep first develop if tone matching fails } } const jpeg = finalShot.encodeToBytes(Skia.ImageFormat.JPEG, 92); if (finalShot !== first) { finalShot.dispose(); } else { first.dispose(); } if (!jpeg?.length) throw new Error('develop produced no bytes'); return jpeg; } catch (err) { // The preview is also the fallback, as it always was: a file whose sensor // will not decode still opens as the camera rendered it. if (preview && preview.length > 10000) return preview; const fallback = await rawThumbnail(bytes).catch(() => null); if (fallback) return fallback; throw err; } finally { try { raw.dispose(); } catch {} } }