library: open a phone's HEIC, and lock the shelves to accounts
A HEIC is the one camera file no engine on this platform will paint: Chromium answers createImageBitmap with InvalidStateError and its WebCodecs take no hvc1 still, and unlike a RAW there is no JPEG preview inside to lift. It is decoded in software instead, by libheif built to wasm, in a worker of its own — so a folder of them reads without the tab stopping for a second, and what leaves the worker is the same JPEG developRaw hands the studio. Nothing downstream knows the difference. Into the studio it goes on the same terms as every other frame: dropped, picked, or opened from the catalogue, no tier asked and the engine's own colour untouched. The frame's date, ISO and position come off the HEIC itself, which is passed along as the bytes the stamps read. The catalogue's tile does not pay the decode: a phone writes a small HEVC copy of its frame as an item of its own (thmb, 320x240 on an iPhone), reachable only by walking the container by hand, and it decodes in milliseconds where the frame takes a second — from the same 256KB the catalogue already reads for a header. The shelves themselves now want an account: a guest gets the note and the way in, and the scan never starts.
This commit is contained in:
@@ -0,0 +1,88 @@
|
||||
// HEIC → JPEG on the client, the way rawDevelop.ts does it for a camera RAW: the
|
||||
// file is opened by the studio as the picture it holds, so a phone's own file
|
||||
// needs no converter in the middle either.
|
||||
//
|
||||
// Unlike a RAW there is nothing to fit: a HEIC is already a rendered 8-bit sRGB
|
||||
// image that the phone made, not sensor data with the camera's picture style to
|
||||
// be matched, so this decode is a decode — no curve, no matrix, no white
|
||||
// balance, and nothing here touches the RAW develop's colour.
|
||||
//
|
||||
// The reading itself is the worker's (see heicWorker.ts): this file is the queue
|
||||
// in front of it, one request at a time, so a folder being read does not start
|
||||
// four wasm decoders.
|
||||
import type { HeicAsk, HeicReply } from './heicWorker';
|
||||
|
||||
// What a picker is told to offer beside the images it knows: iOS file names are
|
||||
// the only sign of the format, and the MIME type for one is not registered
|
||||
// everywhere a file dialog reads.
|
||||
export const HEIC_ACCEPT = '.heic,.heif';
|
||||
|
||||
export function isHeicName(name: string): boolean {
|
||||
return /\.(heic|heif)$/i.test(name);
|
||||
}
|
||||
|
||||
// The answer a request that never got one resolves to: no JPEG, no error worth
|
||||
// printing — the caller has the same nothing in hand as it has for a frame whose
|
||||
// own preview will not decode.
|
||||
const NOTHING: HeicReply = { id: 0 };
|
||||
|
||||
let worker: Worker | null = null;
|
||||
let seq = 0;
|
||||
const waiting = new Map<number, (reply: HeicReply) => void>();
|
||||
|
||||
function answer(reply: HeicReply): void {
|
||||
const done = waiting.get(reply.id);
|
||||
waiting.delete(reply.id);
|
||||
done?.(reply);
|
||||
}
|
||||
|
||||
// The one worker, made when the first HEIC is asked for and kept for the visit.
|
||||
// A worker that dies takes its pending requests with it rather than leaving a
|
||||
// reader on a promise that will never answer, and the next file gets a new one.
|
||||
function lane(): Worker | null {
|
||||
if (worker) return worker;
|
||||
try {
|
||||
const made = new Worker(new URL('./heicWorker.ts', import.meta.url), { type: 'module' });
|
||||
made.onmessage = (event: MessageEvent<HeicReply>) => answer(event.data);
|
||||
made.onerror = () => {
|
||||
for (const id of [...waiting.keys()]) answer({ ...NOTHING, id });
|
||||
made.terminate();
|
||||
if (worker === made) worker = null;
|
||||
};
|
||||
worker = made;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
return worker;
|
||||
}
|
||||
|
||||
function ask(bytes: Uint8Array, thumb: boolean, max: number): Promise<HeicReply> {
|
||||
const target = lane();
|
||||
if (!target) return Promise.resolve(NOTHING);
|
||||
const id = ++seq;
|
||||
return new Promise((resolve) => {
|
||||
waiting.set(id, resolve);
|
||||
// A copy, because the bytes are read again by whoever asked: the catalogue
|
||||
// reads the frame's own date and position off the same slice it just asked
|
||||
// for a tile from.
|
||||
const ask: HeicAsk = { id, bytes: bytes.slice(), thumb, max };
|
||||
target.postMessage(ask, [ask.bytes.buffer]);
|
||||
});
|
||||
}
|
||||
|
||||
// The picture a frame is, as a JPEG — what the studio opens and what a develop
|
||||
// hands it for a RAW. `max` caps the long edge; 0 keeps the file's own size.
|
||||
export async function heicToJpeg(bytes: Uint8Array, max = 0): Promise<Uint8Array> {
|
||||
const reply = await ask(bytes, false, max);
|
||||
if (!reply.jpeg) throw new Error(reply.error ?? 'heic');
|
||||
return reply.jpeg;
|
||||
}
|
||||
|
||||
// The phone's own small copy of the frame, for the catalogue's tiles: the item it
|
||||
// keeps for exactly this, decoded in milliseconds where the frame is a second.
|
||||
// Null when the file carries no such item, which is a frame with no tile rather
|
||||
// than a frame that failed.
|
||||
export async function heicThumbnail(bytes: Uint8Array): Promise<Uint8Array | null> {
|
||||
const reply = await ask(bytes, true, 0);
|
||||
return reply.jpeg ?? null;
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
// The HEIC decode, off the main thread.
|
||||
//
|
||||
// This is the one reading of a phone's file the app has to do in software: no
|
||||
// browser engine on this platform decodes HEVC stills for `createImageBitmap`
|
||||
// (Chromium answers `InvalidStateError`, and its WebCodecs will not take an hvc1
|
||||
// configuration either), and a HEIC carries no JPEG preview to lift the way a
|
||||
// camera RAW does — the frame inside it is one HEVC image, in grid tiles. So the
|
||||
// file is decoded here, by libheif compiled to wasm, and what leaves this worker
|
||||
// is a JPEG — the same thing `developRaw` and `rawThumbnail` hand their callers.
|
||||
// Nothing downstream of this file knows which of the two it has.
|
||||
//
|
||||
// It runs in a worker for the reason the RAW develop runs in its own: this is a
|
||||
// second of wasm for a 12MP frame (measured 1065ms on an iPhone 8 Plus file,
|
||||
// 4032x3024), and a tab that stops painting for a second while it opens a folder
|
||||
// is not an app. One worker is also one decoder — libheif is single-threaded and
|
||||
// this heap is not shared — so the requests queue here instead of racing.
|
||||
//
|
||||
// A tile does not pay that second: a phone writes a small HEVC thumbnail of the
|
||||
// frame as an item of its own (`thmb` on an iPhone), 320x240 on that same file,
|
||||
// and it decodes in 11ms. It is not a top-level image, so libheif's own reader
|
||||
// never returns it and the container is walked by hand below. A file with no
|
||||
// such item gets no tile — the row still lists the frame by name, which is what
|
||||
// a DNG with no preview gets too.
|
||||
import createHeif, { type HeifImage, type HeifImageHandle, type HeifModule } from 'libheif-js/libheif-wasm/libheif-bundle.mjs';
|
||||
|
||||
// The develop's JPEG: the studio works on what a RAW develop gives it, so this
|
||||
// is full size and encoded to be worked on rather than to be looked at.
|
||||
const QUALITY = 0.92;
|
||||
|
||||
export interface HeicAsk {
|
||||
id: number;
|
||||
// The file — whole for a develop, the head of it for a tile: the thumbnail is
|
||||
// in the first kilobytes (measured: 64KB is enough on that same frame, so the
|
||||
// 256KB the catalogue already reads for a header covers it four times over).
|
||||
bytes: Uint8Array;
|
||||
// The file's own thumbnail item when set, the frame itself otherwise.
|
||||
thumb: boolean;
|
||||
// The long edge to encode at; 0 leaves the decoded size alone.
|
||||
max: number;
|
||||
}
|
||||
|
||||
export interface HeicReply {
|
||||
id: number;
|
||||
jpeg?: Uint8Array;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
// One module per worker, made once and kept: it carries a wasm heap, and the
|
||||
// second HEIC of a roll must not pay for the first one's instantiation.
|
||||
let loading: Promise<HeifModule> | null = null;
|
||||
const heif = (): Promise<HeifModule> => (loading ??= createHeif());
|
||||
|
||||
interface Rgba {
|
||||
data: Uint8ClampedArray;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
// libheif's own display call, wrapped: it answers on a later turn of the event
|
||||
// loop, so the promise is what makes the buffer readable, and the handle has to
|
||||
// outlive the call that reads it — every caller here frees the container after
|
||||
// this has resolved, never before.
|
||||
function pixels(image: HeifImage): Promise<Rgba | null> {
|
||||
const width = image.get_width();
|
||||
const height = image.get_height();
|
||||
if (!width || !height) return Promise.resolve(null);
|
||||
const data = new Uint8ClampedArray(width * height * 4);
|
||||
return new Promise((resolve) =>
|
||||
image.display({ data, width, height }, (decoded) => {
|
||||
image.free();
|
||||
resolve(decoded ? { data, width, height } : null);
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
// Whether a call came back with an error. The status these calls carry is
|
||||
// emscripten's enum object, not a number — a successful one is `{value: 0}`,
|
||||
// which is truthy — so the value inside it is what says yes or no. A handle that
|
||||
// came back well has no `code` at all.
|
||||
function failed(result: { code?: { value: number } }): boolean {
|
||||
return !!result.code?.value;
|
||||
}
|
||||
|
||||
// The small HEVC image a phone keeps of its own frame. Its item id is reached
|
||||
// through the primary image's references, then the container is asked for a
|
||||
// handle to that id — the same high-level call the top-level reader uses, so the
|
||||
// colour conversion and the frame's own transforms come out of libheif either way.
|
||||
async function thumbnailItem(m: HeifModule, bytes: Uint8Array): Promise<Rgba | null> {
|
||||
const ctx = m.heif_context_alloc();
|
||||
try {
|
||||
if (failed(m.heif_context_read_from_memory(ctx, bytes))) return null;
|
||||
const primary = m.heif_context_get_primary_image_handle(ctx);
|
||||
if (failed(primary)) return null;
|
||||
|
||||
let id: number;
|
||||
try {
|
||||
const count = m._heif_image_handle_get_number_of_thumbnails(primary.$$.ptr);
|
||||
if (!count) return null;
|
||||
const list = m._malloc(4 * count);
|
||||
try {
|
||||
m._heif_image_handle_get_list_of_thumbnail_IDs(primary.$$.ptr, list, count);
|
||||
id = m.HEAPU32[list >> 2];
|
||||
} finally {
|
||||
m._free(list);
|
||||
}
|
||||
} finally {
|
||||
m.heif_image_handle_release(primary);
|
||||
}
|
||||
|
||||
const handle: HeifImageHandle = m.heif_js_context_get_image_handle(ctx, id);
|
||||
if (failed(handle)) return null;
|
||||
return await pixels(new m.HeifImage(handle));
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
m.heif_context_free(ctx);
|
||||
}
|
||||
}
|
||||
|
||||
// The frame itself, which is the primary image of the container.
|
||||
async function primaryImage(m: HeifModule, bytes: Uint8Array): Promise<Rgba | null> {
|
||||
const images = new m.HeifDecoder().decode(bytes);
|
||||
const image = images.find((i) => i.is_primary()) ?? images[0];
|
||||
return image ? pixels(image) : null;
|
||||
}
|
||||
|
||||
// The pixels as a JPEG, at most `max` on the long edge. The encode happens here
|
||||
// rather than on the page for the usual reason: a 12MP frame is a 48MB readback
|
||||
// and a canvas large enough to hold it, and neither belongs on the thread that
|
||||
// paints.
|
||||
async function encode(rgba: Rgba, max: number): Promise<Uint8Array> {
|
||||
const scale = max > 0 ? Math.min(1, max / Math.max(rgba.width, rgba.height)) : 1;
|
||||
const width = Math.max(1, Math.round(rgba.width * scale));
|
||||
const height = Math.max(1, Math.round(rgba.height * scale));
|
||||
const canvas = new OffscreenCanvas(width, height);
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (!ctx) throw new Error('no 2d context');
|
||||
// The cast is the buffer's own kind: a typed array's default type is one that
|
||||
// may be shared, and `ImageData` takes only one that is not.
|
||||
const bitmap = await createImageBitmap(
|
||||
new ImageData(rgba.data as Uint8ClampedArray<ArrayBuffer>, rgba.width, rgba.height)
|
||||
);
|
||||
ctx.drawImage(bitmap, 0, 0, width, height);
|
||||
bitmap.close();
|
||||
const blob = await canvas.convertToBlob({ type: 'image/jpeg', quality: QUALITY });
|
||||
return new Uint8Array(await blob.arrayBuffer());
|
||||
}
|
||||
|
||||
// The worker's own surface. `self` is typed as the page's window by the DOM lib
|
||||
// the rest of the app compiles against, so the two calls this file makes are
|
||||
// named here instead of pulling a second, conflicting lib in.
|
||||
const scope = self as unknown as {
|
||||
onmessage: ((event: MessageEvent<HeicAsk>) => void) | null;
|
||||
postMessage: (message: HeicReply, transfer?: Transferable[]) => void;
|
||||
};
|
||||
|
||||
scope.onmessage = (event) => {
|
||||
const { id, bytes, thumb, max } = event.data;
|
||||
void (async (): Promise<Rgba | null> => {
|
||||
const m = await heif();
|
||||
return thumb ? thumbnailItem(m, bytes) : primaryImage(m, bytes);
|
||||
})()
|
||||
.then(async (rgba) => {
|
||||
if (!rgba) throw new Error('no image');
|
||||
const jpeg = await encode(rgba, max);
|
||||
scope.postMessage({ id, jpeg }, [jpeg.buffer]);
|
||||
})
|
||||
.catch((err: unknown) => scope.postMessage({ id, error: err instanceof Error ? err.message : String(err) }));
|
||||
};
|
||||
@@ -15,6 +15,7 @@
|
||||
import type { Recipe } from '../../shared/types';
|
||||
import { readCapturedAt } from './imageOps';
|
||||
import { isRawName, rawThumbnail } from './rawDevelop';
|
||||
import { heicThumbnail, isHeicName } from './heicDevelop';
|
||||
import { openAt, walkPass, type Walk, type WalkFile } from './rollWalk';
|
||||
|
||||
const DB_NAME = 'recipescam-library';
|
||||
@@ -898,12 +899,20 @@ export async function scanFolder(
|
||||
}
|
||||
const raw = isRawName(file.name);
|
||||
const isTiff = /\.(tiff?)$/i.test(file.name);
|
||||
// A HEIC is read for its tile the way a RAW is, and reads less: the head of
|
||||
// the file is the whole of what its own thumbnail item needs (see
|
||||
// heicDevelop.ts), so it takes the same slice an ordinary photo does.
|
||||
const heic = isHeicName(file.name);
|
||||
const RAW_HEADER_BYTES = 16 * 1024 * 1024;
|
||||
const sliceBuffer = await file.slice(0, (raw || isTiff) ? Math.min(file.size, RAW_HEADER_BYTES) : HEAD_BYTES).arrayBuffer();
|
||||
const bytes = new Uint8Array(sliceBuffer);
|
||||
const preview = (raw || isTiff) ? await rawThumbnail(bytes, file.name) : null;
|
||||
const src = preview ? new Blob([preview as BlobPart], { type: 'image/jpeg' }) : raw ? null : file;
|
||||
const size = preview ? jpegSize(preview) : raw ? null : jpegSize(bytes);
|
||||
const preview = heic
|
||||
? await heicThumbnail(bytes)
|
||||
: raw || isTiff
|
||||
? await rawThumbnail(bytes, file.name)
|
||||
: null;
|
||||
const src = preview ? new Blob([preview as BlobPart], { type: 'image/jpeg' }) : raw || heic ? null : file;
|
||||
const size = preview ? jpegSize(preview) : raw || heic ? null : jpegSize(bytes);
|
||||
const thumb = src ? await tile(src, size) : null;
|
||||
const taken = (await readCapturedAt(bytes)) ?? file.lastModified;
|
||||
const cut = rel.lastIndexOf('/');
|
||||
|
||||
Reference in New Issue
Block a user