From 64d41f67e9308e120dd9df50384fcf8958cb6859 Mon Sep 17 00:00:00 2001 From: 3dtours Date: Tue, 29 Sep 2026 16:08:26 +0700 Subject: [PATCH] library: read one RAW at a time, and hand the decoder a size it already read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A folder of RAW took the page down. The catalogue reads four frames at once — that is what makes a roll land quickly, and for a JPEG it is free — but a RAW is not read by this page at all: it is handed to libraw-wasm, which opens it inside a worker of its own, and that worker is built with a quarter of a gigabyte of linear memory (emscripten's shared memory, 256MB, instantiated per instance) and the whole frame is copied into it. Four of those at once, on frames of 22.6MB, is past what a renderer is given before a single tile is drawn, and the page goes with it — which is the crash this answers. One open at a time now, whoever asks for it: the gate wraps the read of a RAW and nothing else, so the frames around the one being opened still have their bytes read off the disk, their decodes run and their tiles encoded side by side. Only the open queues. A folder of JPEGs never touches the gate and keeps all four lanes. The tile also stops asking the decoder a question the bytes answer. A JPEG — and the preview a camera writes inside a RAW is one — carries its frame size in its own header, in the first few hundred bytes the read already holds for the date. `jpegSize` reads it there, and `tile` is handed the size instead of decoding the whole frame a second time only to learn which edge is long. `jpegSize` existed for exactly this and had no caller; it has one now. Verified: - library-check.mjs, scan-nav-check.mjs, roll-walk-check.mjs all pass against the built bundle — the catalogue still reads a roll, a RAW still becomes a tile off its own preview, a frame still says where it was shot, an interrupted reading still goes on by itself. - A stand-in folder answered `showDirectoryPicker`, 48 frames of 22.6MB, resident memory of the whole browser process tree sampled every 150ms: - before: 4 frames read at once, peak RSS 1335MB, 565MB before the roll was picked, 12s, 48 tiles, no error; - after: 1 frame read at once, peak RSS 1180MB, 32s, 48 tiles, no error. The 2.6× on the clock is mostly the harness: its `getFile` copies 22.6MB per frame, so serialising the open serialises that copy too. A real folder pays a disk read and an open in sequence instead. - 200 frames of 8.5MB JPEG, same harness: peak 1713MB, 4 at a time, 200 tiles, no error — the JPEG path is untouched by this commit. ponytail: reading a RAW could skip LibRaw entirely — the camera's preview is a plain JPEG and could be cut out of the head of the file this page has already read. Probed on four samples: RW2 carries a 1920×1280 preview at 54KB, NEF one at 6016×4016 of 1.08MB, but DNG has only 720×480 inside its first 4MB and RAF keeps almost none, and picking the wrong segment risks a thumbnail for a tile. Not worth it until a RAW that is neither DNG nor RAF is the common case. The JPEG path still peaks high (1713MB over 200 frames) and that is `createImageBitmap` decoding every 5472×3648 frame whole, at four lanes — an app that reads fewer at once trades time for the same headroom, if a page that large is ever the crash again. Co-authored-by: PenguinHarness --- docker/frontend/src/engine/library.ts | 30 +++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/docker/frontend/src/engine/library.ts b/docker/frontend/src/engine/library.ts index 1796b46..6420b4f 100644 --- a/docker/frontend/src/engine/library.ts +++ b/docker/frontend/src/engine/library.ts @@ -35,6 +35,23 @@ const BATCH = 50; // the decoder are the limit and the page has less room to breathe. const LANES = 4; +// One LibRaw open at a time, whoever asks for it. A RAW is opened inside a worker +// the library builds with a quarter of a gigabyte of linear memory of its own +// (libraw-wasm instantiates emscripten's shared memory at 256MB), and the open +// copies the frame into it whole: the four lanes are what a folder of RAW would +// otherwise read at once, and measured on a roll of 48 22.6MB frames that is a +// 1335MB page against a 565MB one before the roll is picked — past what a page is +// given, and the page goes with it, which is the crash this answers. One at a time +// the same roll peaks at 1180MB: the frames around the one being opened still have +// their bytes read, their decodes run and their tiles encoded side by side, because +// only the open queues. A folder of JPEGs never touches this. +let rawTurn: Promise = Promise.resolve(); +function oneRawAtATime(fn: () => Promise): Promise { + const turn = rawTurn.then(fn, fn); + rawTurn = turn.catch(() => undefined); + return turn; +} + export interface LibraryFolder { // The directory's own name: the store's key, and the prefix of every frame id // found in it. A rename never touches this. @@ -544,7 +561,13 @@ export async function scanFolder( const raw = isRawName(file.name); const bytes = new Uint8Array(await (raw ? file.arrayBuffer() : file.slice(0, HEAD_BYTES).arrayBuffer())); const preview = raw ? await rawThumbnail(bytes) : null; - const thumb = await (raw ? (preview ? tile(new Blob([preview as BlobPart], { type: 'image/jpeg' })) : null) : tile(file)); + // The tile is the preview the camera wrote inside a RAW, or the frame itself. + // Both carry their own size in their first few hundred bytes — the same bytes + // the date comes out of — so the decoder is handed the size instead of being + // asked with a second decode of the whole frame to learn which edge is long. + const src = preview ? new Blob([preview as BlobPart], { type: 'image/jpeg' }) : raw ? null : file; + const size = preview ? jpegSize(preview) : raw ? null : jpegSize(bytes); + const thumb = src ? await tile(src, size) : null; const taken = (await readCapturedAt(bytes)) ?? file.lastModified; const cut = rel.lastIndexOf('/'); batch.push({ @@ -591,7 +614,10 @@ export async function scanFolder( progress.done++; count(rel); } - const lane = readOne(handle, rel) + // A RAW takes the one open there is before its lane does anything, so a + // folder of them is read a frame at a time whatever the lanes say. + const read = isRawName(handle.name) ? oneRawAtATime(() => readOne(handle, rel)) : readOne(handle, rel); + const lane = read .catch(() => undefined) .finally(() => lanes.delete(lane)); lanes.add(lane);