681 lines
30 KiB
TypeScript
681 lines
30 KiB
TypeScript
// RAW → JPEG on the client, so a camera's own file opens in the studio without
|
|
// a DNG converter in the middle (see native_raw_processing_opfs_architecture.md).
|
|
//
|
|
// A RAW opens at the colour the camera chose for it. LibRaw is kept out of white
|
|
// balance and tone (below) and knows nothing of the body's picture style, so the
|
|
// develop is fitted to the one rendering the file does carry with it: the preview
|
|
// the camera wrote inside it, which is the frame the photographer saw and the one
|
|
// a desktop viewer shows. Develop the sensor, take a block grid out of each on
|
|
// the encoded values, fit the cubic a channel between them, and develop the
|
|
// sensor through that curve (see previewMatch.ts).
|
|
//
|
|
// The curve is the whole fit. A 3x3 was least squared ahead of it for a long
|
|
// time and is what put the olive cast on the Olympus frame this was reported on:
|
|
// a develop keeps its luminance and its colour in the same three numbers and the
|
|
// luminance is two orders of magnitude the larger, so a least squares dominated
|
|
// by it holds the channel means and buys them with the colour axis. The camera's
|
|
// picture style is a curve, not a matrix.
|
|
//
|
|
// Measured on that frame, standard deviation of R-G over it (the file's own
|
|
// preview 8.3, the camera's own JPEG 12.3, an olive frame being one where this
|
|
// has gone to nothing): 11.6 as the develop left it, 11.9 through the curve
|
|
// alone, 4.6 through the curve under the fitted 3x3. What no per-pixel transform
|
|
// reaches either way is the camera's own sharpening.
|
|
//
|
|
// Developing the sensor is also what keeps the highlights: 2.3% of the frame at
|
|
// pure white against 3.3% in the preview itself, since the 8-bit preview threw
|
|
// the headroom away and the develop still has it. The preview is therefore the
|
|
// reference and the fallback, not the frame: it is what a file with no usable
|
|
// preview (some DNG), or with sensor data that will not decode, opens as.
|
|
//
|
|
// The develop: LibRaw demosaics in its own worker; what comes back is linear camera data,
|
|
// which this file turns into the sRGB the rest of the pipeline expects. The
|
|
// settings below were checked against the preview a Sony ILME-FX30 writes into
|
|
// its own ARW (the camera JPEG, read straight out of the file): the developed
|
|
// frame and that preview agree to within 1% on both channel ratios, R/G 0.909
|
|
// against 0.903 and B/G 0.603 against 0.606.
|
|
// - noAutoScale + useCameraWb:false + noAutoBright + gamm [1,1] keep LibRaw out
|
|
// of white balance and tone, so `cam_mul` and `rgb_cam` can be applied here
|
|
// exactly once.
|
|
// - halfSize:false: the frame opens at the sensor's own resolution — a 24MP
|
|
// RAW develops to 24MP (6020x4024 on the GR, 6000x4000 on the Fuji), not to
|
|
// the quarter the half-size demosaic reports. `userQual` 3 then demosaics
|
|
// all of it. The develop was half-size until a GR's DNG opened at 3010x2012
|
|
// against its own JPEG's 6000x4000: the quarter-size frame was the flag,
|
|
// not the file. ponytail: costs ~4x the develop time and two full-size F16
|
|
// surfaces; put the flag behind a "draft" toggle if a phone ever has to.
|
|
//
|
|
// The band loop exists because a single Float32 copy of the whole plane would be
|
|
// ~100MB. Each band is decoded, normalised and drawn before the next is read.
|
|
// ponytail: the develop itself is one full-resolution shader draw per band, on
|
|
// the main thread (Skia is not available in the RAW worker). Move it to a worker
|
|
// with an OffscreenCanvas if the develop ever blocks the UI visibly.
|
|
import LibRaw from 'libraw-wasm';
|
|
import { sensorWhite } from './sensorWhite';
|
|
import { f32ToF16 } from './halfFloat';
|
|
import { toneMatch, FLAT_TONE, MATCH_GRID } from './previewMatch';
|
|
import { Skia } from './skiaShim';
|
|
|
|
// What `imageData()` returns for the settings below: 16-bit, 3 channels, black
|
|
// level already gone — the post-process subtracts it whatever `noAutoScale`
|
|
// says, which only holds back the white balance and the output scaling.
|
|
//
|
|
// Its white level is not `maximum` but the frame's own plateau, a factor of 1.89
|
|
// to 2.00 out (see sensorWhite below). The gain carries that level, so the white
|
|
// lands back on 1.0 — and on every body, not just the two that factor two was
|
|
// fitted on: without it every frame opened a stop bright (the FX30's own JPEG has
|
|
// 0.05% of pixels at pure white where the develop had 0.76%) and a body that sat
|
|
// lower (X-Trans, 1.892) never even reached the highlight desaturation, which
|
|
// starts at 0.95 of the sensor.
|
|
const SETTINGS = {
|
|
halfSize: false,
|
|
outputBps: 16,
|
|
outputColor: 0,
|
|
noAutoScale: true,
|
|
useCameraWb: false,
|
|
noAutoBright: true,
|
|
gamm: [1, 1] as [number, number],
|
|
userQual: 3,
|
|
highlight: 2,
|
|
};
|
|
|
|
// 16-bit samples, but everything below is [0,1] — Skia reads an F32 image as
|
|
// already-normalised colour, and a raw 0..65535 plane comes back black.
|
|
const SAMPLE_MAX = 65535;
|
|
// Pixels per band, held under the point where the Float32 copy dominates the
|
|
// memory the page is allowed to use.
|
|
const BAND_PIXELS = 2_000_000;
|
|
|
|
const RAW_DEVELOP_SKSL = `
|
|
uniform shader raw;
|
|
uniform float gain; // 1 / the white level the frame itself ran out at
|
|
uniform float4 mul; // cam_mul, green-normalised
|
|
uniform float4 m0; // camera -> sRGB, the first three columns of rgb_cam
|
|
uniform float4 m1;
|
|
uniform float4 m2;
|
|
uniform float4 crop; // (y offset of this band, 0, 0, 0)
|
|
uniform float4 t0; // the file's own rendering, one cubic a channel on the
|
|
uniform float4 t1; // encoded value: r a b c d per float4 (flat when there is
|
|
uniform float4 t2; // no preview to fit to)
|
|
|
|
// The camera's rendering is a shape, and this is that shape, read on the value
|
|
// the develop left.
|
|
float tone(float4 w, float x) {
|
|
return clamp(w.x + x * (w.y + x * (w.z + x * w.w)), 0.0, 1.0);
|
|
}
|
|
|
|
float3 encode(float3 x) {
|
|
x = clamp(x, 0.0, 1.0);
|
|
return mix(x * 12.92, 1.055 * pow(x, float3(1.0 / 2.4)) - 0.055, step(float3(0.0031308), x));
|
|
}
|
|
|
|
half4 main(float2 pos) {
|
|
float4 p = raw.eval(float2(pos.x, pos.y - crop.x));
|
|
// The plane arrives with the black level already subtracted — it floors at 0,
|
|
// not at color_data.black (measured on the FX30 ARW: the sensor mosaic floors
|
|
// at 334, the plane at 0, a quarter of its red samples under the black level).
|
|
// Subtracting it again drained red and blue — the two channels the gains lift
|
|
// most — and dragged every frame towards green.
|
|
float3 n = max(p.rgb * gain, 0.0); // the sensor's own levels, white level 1.0
|
|
float3 lin = n * mul.rgb;
|
|
// Only the floor there. A photo the sensor could not hold goes over the white
|
|
// level in all three channels, and clipping them one by one before the WB gains
|
|
// is what tints what is left of the highlight: green — the channel the gains are
|
|
// normalised to — stops at 1.0 while red and blue, which need their 2.6x and
|
|
// 1.6x, are already past it, so the blown area comes out magenta. Keep the
|
|
// channel ratios through the matrix instead and let the overflow fade to white.
|
|
float3 rgb = float3(dot(m0.xyz, lin), dot(m1.xyz, lin), dot(m2.xyz, lin));
|
|
float mx = max(max(rgb.r, rgb.g), rgb.b);
|
|
// A pixel that has run to the white level has no colour of its own left to keep,
|
|
// and what the gains made of it is an artefact, not a colour: ease the pixel
|
|
// towards the neutral of its own value as that point is approached. The clip to
|
|
// read is the one the gains make, not only the sensor's own — the gains here are
|
|
// 1.7x and 1.9x on red and blue, so a blown sky reaches the white level at 0.59 of
|
|
// the raw range in those channels while the green, which the gains are
|
|
// normalised to, only reaches it at 1.0. Read on the sensor's levels alone the
|
|
// gate stayed shut across a whole blown sky and left the develop's magenta in it
|
|
// (227,184,245 at the gate's own value against 245,245,245 read where the gains
|
|
// put the clip, the camera's preview white at that block). Both are read, so a
|
|
// body whose gains do not lift a channel keeps the sensor's own clip as its gate.
|
|
// mx is the value to hold: it is the pixel's own lightness.
|
|
float hi = max(max(n.r, n.g), n.b);
|
|
hi = max(hi, max(max(lin.r, lin.g), lin.b));
|
|
float blown = smoothstep(0.99, 1.02, hi);
|
|
rgb = mix(rgb, float3(mx), blown);
|
|
// The overflow used to fade towards white — mix(rgb / mx, 1, 1 - 1 / mx) —
|
|
// which put every pixel of a blown sky on exactly 1.0 and threw the two stops
|
|
// the sensor held above white away with it: LIGHT's HIGHLIGHT row then had a
|
|
// flat white to pull on and nothing to reveal. The white point is moved down
|
|
// instead and the overflow squeezed back in under it by the doc's soft knee:
|
|
//
|
|
// y = T + over / (1 + 2S*over), over = mx - T, S = 1 / (2(1 - T))
|
|
//
|
|
// S is what puts 1.0 on the asymptote, so the sensor's own plateau — two white
|
|
// levels up, see the gain above — lands at ~0.94 and the first stop over white
|
|
// spends 0.85..0.94. Below T the frame is untouched and the curve leaves T with
|
|
// the slope it arrived with (1), so there is no seam to mask; above it the frame
|
|
// darkens, which is the one move no later pass can undo — which is the point,
|
|
// and what the per-file curve below then measures the REST of the frame back from.
|
|
//
|
|
// ponytail: 4.7 stops of headroom now share ~5% of the ramp, and the develop
|
|
// still leaves as an 8-bit JPEG. Give it a float16 output when RAW highlights
|
|
// have to print rather than merely be seen.
|
|
if (mx > 0.7) {
|
|
float over = mx - 0.7;
|
|
rgb *= (0.7 + over / (1.0 + over * 3.3333)) / mx;
|
|
}
|
|
float3 e = encode(rgb);
|
|
// The file's own colour: the curve, in float, on the encoded value it was fitted
|
|
// on — and the last step the frame leaves through. Not the 8-bit colour filter
|
|
// this used to be painted through: the curve carries an exposure (the preview is
|
|
// the reference, so matching its brightness is part of matching its colour), and
|
|
// it is what carries the body's own tone — a 3x3 can only scale, and the one that
|
|
// used to sit here bought the channel means by collapsing the colour axis it was
|
|
// fitted against (see previewMatch.ts). No rolloff: white is one of the values
|
|
// the curve is fitted on, and it is pinned there, so a blown pixel still lands on
|
|
// white while the frame's own highlights stay where the knee above left them.
|
|
float3 o = float3(tone(t0, e.r), tone(t1, e.g), tone(t2, e.b));
|
|
// ...and the neutral the gate above drew is what the curve undoes: one cubic a
|
|
// channel, fitted on a grid that has no block left to fit where the frame ran out
|
|
// (the blown blocks are excluded, see previewMatch), so at the plateau the three
|
|
// curves agree only at 1.0 and part company either side of it — measured on the
|
|
// ORF, a 255,255,255 sky came back 254,255,255 and 253,255,254, red under green
|
|
// across a quarter of the frame. The gate is the develop's own statement that the
|
|
// pixel had no colour of its own, so it is re-read here, on the value that leaves.
|
|
return half4(half3(mix(o, float3(max(max(o.r, o.g), o.b)), blown)), 1.0);
|
|
}
|
|
`;
|
|
|
|
// The extensions LibRaw is compiled for. Not `tif`: a TIFF in a camera folder is
|
|
// as likely to be an export as a RAW.
|
|
const RAW_EXT = [
|
|
'arw', 'srf', 'sr2', 'cr2', 'cr3', 'crw', 'nef', 'nrw', 'dng', 'orf', 'raf',
|
|
'rw2', 'pef', 'srw', 'raw', 'rwl', '3fr', 'fff', 'iiq', 'mos', 'mrw', 'x3f',
|
|
];
|
|
|
|
export const RAW_ACCEPT = RAW_EXT.map((e) => `.${e}`).join(',');
|
|
|
|
export function isRawName(name: string): boolean {
|
|
const ext = name.slice(name.lastIndexOf('.') + 1).toLowerCase();
|
|
return name.includes('.') && RAW_EXT.includes(ext);
|
|
}
|
|
|
|
// The camera's own preview, when the file carries one: the colour reference the
|
|
// curve below is fitted to, and what the file opens as when the sensor does not
|
|
// decode.
|
|
// ponytail: it is 1616x1080 on an A5100 and 1620x1080 on an FX30, so handing it
|
|
// back is a 1.7MP frame — a print past it has to come off the develop, which is
|
|
// what a developed file already gives.
|
|
async function cameraPreview(raw: LibRaw): Promise<Uint8Array | null> {
|
|
const thumb = await raw.thumbnailData().catch(() => undefined);
|
|
if (thumb?.format !== 'jpeg' || !thumb.data?.length) return null;
|
|
return new Uint8Array(thumb.data);
|
|
}
|
|
|
|
// Reads EXIF Orientation tag (1-8) from JPEG byte headers.
|
|
export function getJpegOrientation(bytes: Uint8Array): number {
|
|
if (bytes.length < 12 || bytes[0] !== 0xff || bytes[1] !== 0xd8) return 1;
|
|
let offset = 2;
|
|
while (offset < bytes.length - 4) {
|
|
if (bytes[offset] !== 0xff) { offset++; continue; }
|
|
const marker = bytes[offset + 1];
|
|
if (marker === 0xe1) {
|
|
const length = (bytes[offset + 2] << 8) | bytes[offset + 3];
|
|
if (offset + 4 + length > bytes.length) break;
|
|
const exifHeader = String.fromCharCode(...bytes.subarray(offset + 4, offset + 10));
|
|
if (exifHeader === 'Exif\0\0') {
|
|
const tiffStart = offset + 10;
|
|
const littleEndian = bytes[tiffStart] === 0x49 && bytes[tiffStart + 1] === 0x49;
|
|
const read16 = (o: number) => littleEndian ? (bytes[o] | (bytes[o + 1] << 8)) : ((bytes[o] << 8) | bytes[o + 1]);
|
|
const read32 = (o: number) => littleEndian ? (bytes[o] | (bytes[o + 1] << 8) | (bytes[o + 2] << 16) | (bytes[o + 3] << 24)) : ((bytes[o] << 24) | (bytes[o + 1] << 16) | (bytes[o + 2] << 8) | bytes[o + 3]);
|
|
if (read16(tiffStart + 2) !== 0x002a) break;
|
|
const firstIfdOffset = read32(tiffStart + 4);
|
|
let dirStart = tiffStart + firstIfdOffset;
|
|
if (dirStart + 2 > bytes.length) break;
|
|
const entries = read16(dirStart);
|
|
dirStart += 2;
|
|
for (let i = 0; i < entries; i++) {
|
|
const entryOffset = dirStart + i * 12;
|
|
if (entryOffset + 12 > bytes.length) break;
|
|
const tag = read16(entryOffset);
|
|
if (tag === 0x0112) {
|
|
const val = read16(entryOffset + 8);
|
|
return val >= 1 && val <= 8 ? val : 1;
|
|
}
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
if (marker === 0xda || marker === 0xd9) break;
|
|
const segLen = (bytes[offset + 2] << 8) | bytes[offset + 3];
|
|
if (segLen < 2) break;
|
|
offset += 2 + segLen;
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
// Fast pure JS binary scanner to extract embedded JPEG preview from RAW files (ARW, NEF, CR2, CR3, DNG, RAF, RW2, ORF, PEF).
|
|
// Scans for all embedded JPEGs in the RAW file and returns the largest segment (the full-size camera preview).
|
|
export function extractEmbeddedJpeg(buf: Uint8Array): Uint8Array | null {
|
|
const len = buf.length;
|
|
let largest: Uint8Array | null = null;
|
|
let maxLen = 0;
|
|
|
|
for (let i = 0; i < len - 4; i++) {
|
|
if (buf[i] === 0xff && buf[i + 1] === 0xd8 && buf[i + 2] === 0xff) {
|
|
const start = i;
|
|
let pos = i + 2;
|
|
let eoiPos = -1;
|
|
let validJpeg = true;
|
|
|
|
while (pos < len - 4) {
|
|
if (buf[pos] !== 0xff) {
|
|
validJpeg = false;
|
|
break;
|
|
}
|
|
const m = buf[pos + 1];
|
|
if (m === 0xd9) {
|
|
eoiPos = pos + 2;
|
|
break;
|
|
}
|
|
if (m === 0xda) {
|
|
const sosHeaderLen = (buf[pos + 2] << 8) | buf[pos + 3];
|
|
if (sosHeaderLen < 2 || pos + 2 + sosHeaderLen > len) {
|
|
validJpeg = false;
|
|
break;
|
|
}
|
|
let scanPos = pos + 2 + sosHeaderLen;
|
|
const searchEnd = Math.min(len - 1, scanPos + 20 * 1024 * 1024);
|
|
while (scanPos < searchEnd) {
|
|
if (buf[scanPos] === 0xff) {
|
|
const marker = buf[scanPos + 1];
|
|
if (marker === 0xd9) {
|
|
eoiPos = scanPos + 2;
|
|
break;
|
|
}
|
|
if (marker === 0x00 || (marker >= 0xd0 && marker <= 0xd7)) {
|
|
scanPos += 2;
|
|
continue;
|
|
}
|
|
}
|
|
scanPos++;
|
|
}
|
|
break;
|
|
}
|
|
if (m === 0x00 || (m >= 0xd0 && m <= 0xd7)) {
|
|
pos += 2;
|
|
continue;
|
|
}
|
|
const segLen = (buf[pos + 2] << 8) | buf[pos + 3];
|
|
if (segLen < 2 || pos + 2 + segLen > len) {
|
|
validJpeg = false;
|
|
break;
|
|
}
|
|
pos += 2 + segLen;
|
|
}
|
|
|
|
if (eoiPos > start && validJpeg) {
|
|
const segSize = eoiPos - start;
|
|
if (segSize > 10000 && segSize > maxLen) {
|
|
maxLen = segSize;
|
|
largest = buf.subarray(start, eoiPos);
|
|
}
|
|
i = eoiPos - 1;
|
|
}
|
|
}
|
|
}
|
|
return largest;
|
|
}
|
|
|
|
// The preview on its own, for the catalogue: a folder of RAW files has to show
|
|
// a tile per frame, and unpack_thumb is a seek and a copy where the develop
|
|
// above is a full decode of every pixel at full resolution. No preview inside
|
|
// the file means no tile — the row still lists the frame by name, and the
|
|
// studio develops it the moment it is opened.
|
|
export function tiffThumbnail(bytes: Uint8Array): Uint8Array | null {
|
|
try {
|
|
const img = Skia.Image.MakeImageFromEncoded(bytes);
|
|
if (!img) return null;
|
|
try {
|
|
const w = img.width();
|
|
const h = img.height();
|
|
if (!w || !h) return null;
|
|
const n = 640;
|
|
const wide = w >= h;
|
|
const targetW = wide ? n : Math.max(1, Math.round((w / h) * n));
|
|
const targetH = wide ? Math.max(1, Math.round((h / w) * n)) : n;
|
|
|
|
const surface = Skia.Surface.MakeOffscreen(targetW, targetH) ?? Skia.Surface.Make(targetW, targetH);
|
|
if (!surface) return null;
|
|
const canvas = surface.getCanvas();
|
|
canvas.drawImageRectCubic(
|
|
img,
|
|
Skia.XYWHRect(0, 0, w, h),
|
|
Skia.XYWHRect(0, 0, targetW, targetH),
|
|
1 / 3,
|
|
1 / 3
|
|
);
|
|
surface.flush();
|
|
const snapshot = surface.makeImageSnapshot();
|
|
surface.dispose();
|
|
if (!snapshot) return null;
|
|
const jpeg = snapshot.encodeToBytes(Skia.ImageFormat.JPEG, 80);
|
|
snapshot.dispose();
|
|
return jpeg ? new Uint8Array(jpeg) : null;
|
|
} finally {
|
|
img.delete();
|
|
}
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// The preview on its own, for the catalogue: a folder of RAW files has to show
|
|
// a tile per frame, and unpack_thumb is a seek and a copy where the develop
|
|
// above is a full decode of every pixel at full resolution. No preview inside
|
|
// the file means no tile — the row still lists the frame by name, and the
|
|
// studio develops it the moment it is opened.
|
|
export async function rawThumbnail(bytes: Uint8Array, fileName?: string): Promise<Uint8Array | null> {
|
|
const fast = extractEmbeddedJpeg(bytes);
|
|
if (fast) return fast;
|
|
|
|
if (fileName && /\.(tiff?)$/i.test(fileName)) {
|
|
const tiff = tiffThumbnail(bytes);
|
|
if (tiff) return tiff;
|
|
}
|
|
|
|
const raw = new LibRaw();
|
|
try {
|
|
await raw.open(bytes as unknown as BufferSource, SETTINGS);
|
|
return await cameraPreview(raw);
|
|
} catch {
|
|
return tiffThumbnail(bytes);
|
|
} finally {
|
|
try {
|
|
raw.dispose();
|
|
} catch {}
|
|
}
|
|
}
|
|
|
|
// MATCH_GRID x MATCH_GRID block colours of a frame, one byte per channel.
|
|
function gridOf(image: any, n = MATCH_GRID): Uint8Array | null {
|
|
const surface = Skia.Surface.MakeOffscreen(n, n) ?? Skia.Surface.Make(n, n);
|
|
if (!surface) return null;
|
|
const canvas = surface.getCanvas();
|
|
// Cubic, not a linear tap: this is a 45x reduction and linear reads a handful
|
|
// of source pixels per block — noise for the curve to be fitted on.
|
|
canvas.drawImageRectCubic(
|
|
image,
|
|
Skia.XYWHRect(0, 0, image.width(), image.height()),
|
|
Skia.XYWHRect(0, 0, n, n),
|
|
1 / 3,
|
|
1 / 3
|
|
);
|
|
surface.flush();
|
|
const px = canvas.readPixels(0, 0, {
|
|
width: n,
|
|
height: n,
|
|
colorType: Skia.ColorType.RGBA_8888,
|
|
alphaType: Skia.AlphaType.Unpremul,
|
|
colorSpace: Skia.ColorSpace.SRGB,
|
|
}) as Uint8Array | null;
|
|
surface.dispose();
|
|
return px ? new Uint8Array(px.buffer, px.byteOffset, px.byteLength) : null;
|
|
}
|
|
|
|
// The same grid out of the preview, which the file carries as a JPEG. Decoded and
|
|
// reduced through the same Skia call as the develop's own grid, because the two
|
|
// grids are only comparable — and a curve between them only meaningful — when one
|
|
// resampler made both. A 2D canvas here instead left the curve following its own
|
|
// smoothing: on the A5100 frame the same develop scored dE00 4.7 against 4.4, and
|
|
// the dark end of the frame came out 6 L further from the preview than the curve
|
|
// it was handed asked for.
|
|
function previewGrid(jpeg: Uint8Array, w: number, h: number, n = MATCH_GRID): Uint8Array | null {
|
|
const bmp = Skia.Image.MakeImageFromEncoded(jpeg);
|
|
if (!bmp) return null;
|
|
try {
|
|
if (bmp.width() < n || bmp.height() < n) return null;
|
|
const orient = getJpegOrientation(jpeg);
|
|
const isSwapped = orient === 5 || orient === 6 || orient === 7 || orient === 8;
|
|
const orientedW = isSwapped ? bmp.height() : bmp.width();
|
|
const orientedH = isSwapped ? bmp.width() : bmp.height();
|
|
|
|
const targetAspect = w / h;
|
|
const bmpAspect = orientedW / orientedH;
|
|
let cropW = orientedW;
|
|
let cropH = orientedH;
|
|
if (Math.abs(bmpAspect - targetAspect) > 0.01) {
|
|
if (bmpAspect > targetAspect) {
|
|
cropW = Math.round(orientedH * targetAspect);
|
|
} else {
|
|
cropH = Math.round(orientedW / targetAspect);
|
|
}
|
|
}
|
|
const cropX = Math.round((orientedW - cropW) / 2);
|
|
const cropY = Math.round((orientedH - cropH) / 2);
|
|
|
|
const surface = Skia.Surface.MakeOffscreen(n, n) ?? Skia.Surface.Make(n, n);
|
|
if (!surface) return null;
|
|
const canvas = surface.getCanvas();
|
|
canvas.save();
|
|
|
|
if (orient === 3) {
|
|
canvas.translate(n, n);
|
|
canvas.rotate(180, 0, 0);
|
|
canvas.drawImageRectCubic(
|
|
bmp,
|
|
Skia.XYWHRect(bmp.width() - cropX - cropW, bmp.height() - cropY - cropH, cropW, cropH),
|
|
Skia.XYWHRect(0, 0, n, n),
|
|
1 / 3,
|
|
1 / 3
|
|
);
|
|
} else if (orient === 6) {
|
|
canvas.translate(n, 0);
|
|
canvas.rotate(90, 0, 0);
|
|
canvas.drawImageRectCubic(
|
|
bmp,
|
|
Skia.XYWHRect(cropY, bmp.height() - cropX - cropW, cropH, cropW),
|
|
Skia.XYWHRect(0, 0, n, n),
|
|
1 / 3,
|
|
1 / 3
|
|
);
|
|
} else if (orient === 8) {
|
|
canvas.translate(0, n);
|
|
canvas.rotate(-90, 0, 0);
|
|
canvas.drawImageRectCubic(
|
|
bmp,
|
|
Skia.XYWHRect(bmp.width() - cropY - cropH, cropX, cropH, cropW),
|
|
Skia.XYWHRect(0, 0, n, n),
|
|
1 / 3,
|
|
1 / 3
|
|
);
|
|
} else {
|
|
canvas.drawImageRectCubic(
|
|
bmp,
|
|
Skia.XYWHRect(cropX, cropY, cropW, cropH),
|
|
Skia.XYWHRect(0, 0, n, n),
|
|
1 / 3,
|
|
1 / 3
|
|
);
|
|
}
|
|
canvas.restore();
|
|
surface.flush();
|
|
const px = canvas.readPixels(0, 0, {
|
|
width: n,
|
|
height: n,
|
|
colorType: Skia.ColorType.RGBA_8888,
|
|
alphaType: Skia.AlphaType.Unpremul,
|
|
colorSpace: Skia.ColorSpace.SRGB,
|
|
}) as Uint8Array | null;
|
|
surface.dispose();
|
|
return px ? new Uint8Array(px.buffer, px.byteOffset, px.byteLength) : null;
|
|
} finally {
|
|
bmp.delete();
|
|
}
|
|
}
|
|
|
|
// The camera -> sRGB matrix, applied to the WB'd sensor triple.
|
|
//
|
|
// `rgb_cam` is dcraw's own: it is the camera -> sRGB transform already, its rows
|
|
// sum to one, and the 1/num dcraw scales it by rides in `pre_mul`. Dividing it by
|
|
// pre_mul a second time undoes the normalisation and drags the frame towards
|
|
// green. It is what every body LibRaw knows a `cam_xyz` for gets — the Olympus
|
|
// ORF this was reported on included, whose `rgb_cam` and `cam_xyz` disagree
|
|
// (measured: rgb_cam x cam_mul lands on R/G 1.021 B/G 0.920 against the file's own
|
|
// preview's 1.013 / 0.841, the cam_xyz chain this used to prefer on 0.921 / 0.886
|
|
// — green, and blue for a scene the camera left neutral).
|
|
//
|
|
// `cam_xyz` stays as the fallback for a file that has one and no `rgb_cam`: its
|
|
// rows are the XYZ of each camera channel, so sRGB = xyz_rgb^-1 . cam_xyz^T . cam,
|
|
// with the rows normalised so a neutral camera triple opens neutral, as dcraw's
|
|
// cam_xyz_coeff normalises them.
|
|
const XYZ_TO_SRGB = [
|
|
[3.2404542, -1.5371385, -0.4985314],
|
|
[-0.9692660, 1.8760108, 0.0415560],
|
|
[0.0556434, -0.2040259, 1.0572252],
|
|
];
|
|
|
|
function getCamToSrgbMatrix(cd: any): number[][] {
|
|
if (cd?.rgb_cam && Array.isArray(cd.rgb_cam) && cd.rgb_cam.length >= 3) {
|
|
return [0, 1, 2].map((i) => [0, 1, 2].map((j) => cd.rgb_cam[i]?.[j] || 0));
|
|
}
|
|
|
|
if (cd?.cam_xyz && Array.isArray(cd.cam_xyz) && cd.cam_xyz.length >= 3) {
|
|
const cx = cd.cam_xyz;
|
|
const scale = (Math.abs(cx[0][0]) > 100) ? 10000.0 : 1.0;
|
|
const m = [0, 1, 2].map((i) =>
|
|
[0, 1, 2].map((j) =>
|
|
XYZ_TO_SRGB[i][0] * ((cx[j]?.[0] || 0) / scale) +
|
|
XYZ_TO_SRGB[i][1] * ((cx[j]?.[1] || 0) / scale) +
|
|
XYZ_TO_SRGB[i][2] * ((cx[j]?.[2] || 0) / scale)
|
|
)
|
|
);
|
|
return m.map((row) => {
|
|
const num = row[0] + row[1] + row[2];
|
|
return Math.abs(num) < 1e-6 ? row : row.map((v) => v / num);
|
|
});
|
|
}
|
|
|
|
return [[1, 0, 0], [0, 1, 0], [0, 0, 1]];
|
|
}
|
|
|
|
export async function developRaw(bytes: Uint8Array): Promise<Uint8Array> {
|
|
const raw = new LibRaw();
|
|
let preview: Uint8Array | null = null;
|
|
try {
|
|
// Extract largest embedded JPEG preview from file bytes first
|
|
preview = extractEmbeddedJpeg(bytes);
|
|
// LibRaw copies the buffer it is handed, so the caller's bytes stay intact.
|
|
await raw.open(bytes as unknown as BufferSource, SETTINGS);
|
|
if (!preview) {
|
|
preview = await cameraPreview(raw);
|
|
}
|
|
const meta = await raw.metadata(true);
|
|
const img = await raw.imageData();
|
|
const cd = meta?.color_data;
|
|
if (!img || (!cd?.cam_mul && !cd?.pre_mul) || (!cd.rgb_cam && !cd.cam_xyz)) throw new Error('RAW has no colour data');
|
|
|
|
const { width: w, height: h } = img;
|
|
const data = img.data as Uint16Array;
|
|
if (!w || !h) throw new Error('RAW decoded to nothing');
|
|
|
|
const effect = Skia.RuntimeEffect.Make(RAW_DEVELOP_SKSL);
|
|
if (!effect) throw new Error('develop shader failed to compile');
|
|
|
|
const rawMul = (cd?.cam_mul && cd.cam_mul.some((v: number) => v > 0 && v !== 1))
|
|
? cd.cam_mul
|
|
: (cd?.pre_mul && cd.pre_mul.some((v: number) => v > 0 && v !== 1))
|
|
? cd.pre_mul
|
|
: [2.0, 1.0, 1.5, 1.0];
|
|
const green = rawMul[1] || rawMul[3] || 1;
|
|
const mul = rawMul.map((v: number) => v / green);
|
|
const [r0, r1, r2] = getCamToSrgbMatrix(cd);
|
|
|
|
const bandH = Math.max(1, Math.min(h, Math.floor(BAND_PIXELS / w)));
|
|
const f32 = new Float32Array(w * bandH * 4);
|
|
// The band goes up as half, not float32: the GPU backend puts an F32 image
|
|
// on the 1/255 grid and the shadows quantise to black (see halfFloat.ts).
|
|
const half = new Uint16Array(w * bandH * 4);
|
|
// Every uniform but the crop and the curve, which are the two the passes change:
|
|
// the shader's own order is gain, mul, rgb_cam, crop, then the curve.
|
|
const uniforms = new Float32Array(33);
|
|
uniforms[0] = SAMPLE_MAX / sensorWhite(data, cd.maximum, cd.black);
|
|
uniforms.set([mul[0], mul[1], mul[2], 0, r0[0], r0[1], r0[2], 0, r1[0], r1[1], r1[2], 0, r2[0], r2[1], r2[2], 0], 1);
|
|
|
|
// One develop of the frame, band by band, through `tone` when the file carried a
|
|
// preview to fit a curve to. A function because the frame is drawn twice: once
|
|
// as the sensor left it, to fit that curve against, and then again through it.
|
|
// ponytail: two full band passes, on the main thread. Give develop an F16
|
|
// intermediate (one develop, one colour pass) if the second pass ever shows.
|
|
const develop = (tone: Float32Array | null) => {
|
|
const surface = Skia.Surface.MakeOffscreen(w, h) ?? Skia.Surface.Make(w, h);
|
|
if (!surface) return null;
|
|
uniforms.set(tone ?? FLAT_TONE, 21);
|
|
for (let y0 = 0; y0 < h; y0 += bandH) {
|
|
const rows = Math.min(bandH, h - y0);
|
|
let o = 0;
|
|
for (let i = y0 * w * 3, end = (y0 + rows) * w * 3; i < end; i += 3) {
|
|
f32[o++] = data[i] / SAMPLE_MAX;
|
|
f32[o++] = data[i + 1] / SAMPLE_MAX;
|
|
f32[o++] = data[i + 2] / SAMPLE_MAX;
|
|
f32[o++] = 1;
|
|
}
|
|
f32ToF16(f32, half, w * rows * 4);
|
|
const band = Skia.Image.MakeImage(
|
|
{ width: w, height: rows, colorType: Skia.ColorType.RGBA_F16, alphaType: Skia.AlphaType.Unpremul },
|
|
new Uint8Array(half.buffer, 0, w * rows * 8),
|
|
w * 8
|
|
);
|
|
if (!band) throw new Error('band image failed');
|
|
const child = band.makeShaderOptions(
|
|
Skia.TileMode.Clamp,
|
|
Skia.TileMode.Clamp,
|
|
Skia.FilterMode.Nearest,
|
|
Skia.MipmapMode.None
|
|
);
|
|
uniforms[17] = y0;
|
|
const shader = effect.makeShaderWithChildren(uniforms, [child]);
|
|
const paint = Skia.Paint();
|
|
paint.setShader(shader);
|
|
surface.getCanvas().drawRect(Skia.XYWHRect(0, y0, w, rows), paint);
|
|
surface.flush();
|
|
paint.delete();
|
|
shader.delete();
|
|
child.delete();
|
|
band.delete();
|
|
}
|
|
|
|
const shot = surface.makeImageSnapshot();
|
|
surface.dispose();
|
|
return shot;
|
|
};
|
|
|
|
// The file's own colour: a grid of the develop as it stands, the same grid out
|
|
// of the preview the camera wrote into the file, and the curve between them —
|
|
// which the second develop then draws in the shader. No preview, no curve: the
|
|
// frame opens as the sensor left it.
|
|
const first = develop(null);
|
|
if (!first) throw new Error('no surface for the develop');
|
|
const blocks = preview ? gridOf(first) : null;
|
|
const ref = blocks ? previewGrid(preview as Uint8Array, first.width(), first.height()) : null;
|
|
const tone = blocks && ref ? toneMatch(blocks, ref) : null;
|
|
const matched = tone ? develop(tone) : null;
|
|
const jpeg = (matched ?? first).encodeToBytes(Skia.ImageFormat.JPEG, 92);
|
|
(matched ?? first).dispose();
|
|
if (matched) first.dispose();
|
|
if (!jpeg?.length) throw new Error('develop produced no bytes');
|
|
return jpeg;
|
|
} catch (err) {
|
|
// The preview still opens the file when the sensor will not: a RAW whose
|
|
// colour data is missing (some DNG) is not a RAW that cannot be shown.
|
|
if (preview) return preview;
|
|
const fallback = await rawThumbnail(bytes).catch(() => null);
|
|
if (fallback) return fallback;
|
|
throw err;
|
|
} finally {
|
|
try {
|
|
raw.dispose();
|
|
} catch {}
|
|
}
|
|
}
|