Files
RecipesCam/docker/frontend/src/engine/library.ts
T

1755 lines
71 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// The local catalogue: the folders a visitor has handed over, a thumbnail of
// every frame found in them, and the recipe each frame was last graded with.
// Nothing here leaves the machine — no upload, no server, no account.
//
// The three stores are what the folder is made of: `folders` keeps the directory
// handle so the catalogue survives a reload without a second trip through the
// picker, `photos` keeps a file handle (the RAW itself is read only when a frame
// is opened) plus the small JPEG the grid paints, and `edits` keeps the recipe a
// frame was left at, keyed by the same id the grid uses.
//
// ponytail: Dexie would wrap this in three lines, but IndexedDB stores a
// FileSystemHandle by itself and the whole surface is seven calls — a dependency
// does not pay for itself here. Revisit if the catalogue grows queries (tags,
// smart collections) that hand-rolled cursors would make ugly.
import type { Recipe } from '../../shared/types';
import { readCapturedAt } from './imageOps';
import { isRawName, rawThumbnail, tiffThumbnail } from './rawDevelop';
import { heicThumbnail, heicToJpeg, isHeicName } from './heicDevelop';
import { openAt, walkPass, type Walk, type WalkFile } from './rollWalk';
const DB_NAME = 'recipescam-library';
const FOLDERS = 'folders';
const PHOTOS = 'photos';
const THUMBS = 'thumbs';
const EDITS = 'edits';
const DB_VERSION = 2;
// The tile the catalogue keeps is the same tile the grid cell and the strip both
// paint, and there is one of them per frame of a roll that reaches six figures.
// At 512 on the long edge a tile weighed 42KB and a hundred and sixty thousand of
// them weighed six and a half gigabytes of the reader's disk — for a grid cell
// that is ~240px wide and a strip tile that is 132. A quarter of the pixels and a
// lower quality put the same picture in the same cell: the tile is now ~12KB, and
// the roll a quarter of the disk it was. The quality is the one Lightroom keeps
// its own grid previews at, and at this size the eye cannot tell it from 0.75.
const THUMB_MAX = 640;
const THUMB_QUALITY = 0.8;
// Frames go down in batches, so a 3000-file folder is 60 transactions rather
// than 3000 of them.
const BATCH = 50;
// Frames read at once. Reading a frame is nearly all waiting — the bytes come off
// the disk, the decode runs on a thread of its own — and a wait that is not spent
// on the next frame is time the roll does not get back. Past a few, the disk and
// the decoder are the limit and the page has less room to breathe.
const LANES = 4;
// How often the walk's position is written down, in the same seconds as the
// catalogue is read back on: the position is the whole of what the walk has found
// and not read, and writing it is a string of a few megabytes on a long roll.
const WALK_MS = 5000;
// How often the screen is told how far the reading has come: the column is
// rebuilt out of the count, and a rebuild a frame is a reading that spends its
// time drawing itself. Five times a second is a counter that moves to the eye
// and a page that is doing the reading instead.
const TELL_MS = 200;
// One LibRaw open at a time, whoever asks for it. A RAW is opened inside a worker
// the library builds with a quarter of a gigabyte of linear memory of its own
// (libraw-wasm instantiates emscripten's shared memory at 256MB), and the open
// copies the frame into it whole: the four lanes are what a folder of RAW would
// otherwise read at once, and measured on a roll of 48 22.6MB frames that is a
// 1335MB page against a 565MB one before the roll is picked — past what a page is
// given, and the page goes with it, which is the crash this answers. One at a time
// the same roll peaks at 1180MB: the frames around the one being opened still have
// their bytes read, their decodes run and their tiles encoded side by side, because
// only the open queues. A folder of JPEGs never touches this.
let rawTurn: Promise<unknown> = Promise.resolve();
function oneRawAtATime<T>(fn: () => Promise<T>): Promise<T> {
const turn = rawTurn.then(fn, fn);
rawTurn = turn.catch(() => undefined);
return turn;
}
export interface LibraryFolder {
// The directory's own name: the store's key, and the prefix of every frame id
// found in it. A rename never touches this.
name: string;
// What the tree paints instead, when the visitor has renamed the folder.
label?: string;
handle: FileSystemDirectoryHandle;
}
export interface LibraryPhoto {
// `${folder}/${dir}/${name}` — one frame appears once in the catalogue, under
// the folder it was found in.
id: string;
folder: string;
// The subfolder it sits in, relative to the picked folder — '' at the root.
// This is what the tree column reads.
dir: string;
name: string;
handle: FileSystemFileHandle;
thumb: Blob | null;
// When the shutter fired, per EXIF, else the file's own lastModified.
taken: number;
size: number;
// When the file itself was last written, as the browser reports it. This — with
// the size — is what says a frame has not moved since the last scan and does not
// have to be read again; `taken` is the shutter's own time and says nothing
// about the file. A frame stored before this field existed has none, and is
// read one last time.
mtime?: number;
// The reader's own score for the frame, 0 or absent when they have not given
// it one. It belongs to the catalogue and not to the disk — nothing about the
// file changes — so a rescan carries it over, the way it carries `addedAt`.
star?: number;
// The quarter turn the reader gave the frame so it stands the way they saw
// it, 0 or absent for a frame they have not turned. Like the score it belongs
// to the catalogue rather than to the disk — nothing about the file changes —
// and a rescan carries it over; the studio opens a frame at this angle.
rot?: 0 | 90 | 180 | 270;
addedAt: number;
}
// A folder a scan has walked into, whether or not a frame has been read out of
// it yet: the column draws a roll from these before its frames arrive. They are
// the scan's own report and live no longer than it does — a reload draws the
// folders that hold frames, and the next scan names the rest again.
export interface LibraryDir {
// `${folder}/${rel}` — the tree key, the same way a frame id is spelled.
id: string;
folder: string;
// The path under the picked folder, no leading or trailing slash.
rel: string;
}
export interface LibraryEdit {
photoId: string;
recipe: Recipe;
updatedAt: number;
}
export const SUPPORTED = /\.(jpe?g|png|webp|heic|heif|tiff?|avif)$/i;
export function isSupportedPhoto(name: string): boolean {
return isRawName(name) || SUPPORTED.test(name);
}
export function normPath(p: string): string {
return (p || '').replace(/\\/g, '/').replace(/\/+/g, '/').replace(/\/$/, '');
}
export function photoId(folder: string, name: string): string {
return `${normPath(folder)}/${normPath(name)}`;
}
// The picker is Chromium-only (Chrome, Edge, Opera, Brave): Firefox and Safari
// have no way to hand a page a folder that outlives the visit, and the caller
// says so instead of showing a button that cannot work.
export function canBrowseFolders(): boolean {
return typeof window !== 'undefined' && 'showDirectoryPicker' in window;
}
const THUMB_CACHE_MAX = 800;
const thumbCache = new Map<string, Blob>();
export function cacheThumb(id: string, blob: Blob): void {
if (thumbCache.has(id)) thumbCache.delete(id);
else if (thumbCache.size >= THUMB_CACHE_MAX) {
const first = thumbCache.keys().next().value;
if (first) thumbCache.delete(first);
}
thumbCache.set(id, blob);
}
export function getCachedThumb(id: string): Blob | null {
const blob = thumbCache.get(id);
if (blob) {
thumbCache.delete(id);
thumbCache.set(id, blob);
return blob;
}
return null;
}
async function migrateLegacyDatabase(db: IDBDatabase): Promise<void> {
if (typeof window === 'undefined') return;
const MIGRATED_KEY = 'recipescam-migrated-v2';
if (localStorage.getItem(MIGRATED_KEY) === 'true') return;
if (!db.objectStoreNames.contains(THUMBS) || !db.objectStoreNames.contains(PHOTOS)) return;
try {
await new Promise<void>((resolve) => {
const tx = db.transaction([PHOTOS, THUMBS], 'readwrite');
const photoStore = tx.objectStore(PHOTOS);
const thumbStore = tx.objectStore(THUMBS);
const req = photoStore.openCursor();
req.onsuccess = () => {
const cursor = req.result;
if (cursor) {
const val = cursor.value as any;
if (val && val.thumb) {
const thumbBlob = val.thumb as Blob;
const id = val.id as string;
cacheThumb(id, thumbBlob);
thumbStore.put({ id, thumb: thumbBlob });
delete val.thumb;
cursor.update(val);
}
cursor.continue();
} else {
resolve();
}
};
req.onerror = () => resolve();
tx.oncomplete = () => resolve();
});
localStorage.setItem(MIGRATED_KEY, 'true');
} catch {
// Non-fatal if migration is interrupted
}
}
let dbPromise: Promise<IDBDatabase> | null = null;
function openDb(): Promise<IDBDatabase> {
dbPromise ??= new Promise<IDBDatabase>((resolve, reject) => {
const req = indexedDB.open(DB_NAME, DB_VERSION);
req.onupgradeneeded = () => {
const db = req.result;
if (!db.objectStoreNames.contains(FOLDERS)) db.createObjectStore(FOLDERS, { keyPath: 'name' });
if (!db.objectStoreNames.contains(PHOTOS)) {
const photos = db.createObjectStore(PHOTOS, { keyPath: 'id' });
photos.createIndex('folder', 'folder');
photos.createIndex('taken', 'taken');
}
if (!db.objectStoreNames.contains(THUMBS)) {
db.createObjectStore(THUMBS, { keyPath: 'id' });
}
if (!db.objectStoreNames.contains(EDITS)) db.createObjectStore(EDITS, { keyPath: 'photoId' });
};
req.onsuccess = () => {
const db = req.result;
void migrateLegacyDatabase(db).then(() => resolve(db));
};
req.onerror = () => reject(req.error);
});
return dbPromise;
}
export async function findFolderRelationship(
newHandle: FileSystemDirectoryHandle,
existingFolders: LibraryFolder[]
): Promise<
| { type: 'subfolder'; parent: LibraryFolder; relPath: string }
| { type: 'parent'; child: LibraryFolder }
| { type: 'root' }
> {
for (const existing of existingFolders) {
try {
const relParts = await existing.handle.resolve(newHandle);
if (relParts && relParts.length > 0) {
return { type: 'subfolder', parent: existing, relPath: relParts.join('/') };
}
} catch {}
}
for (const existing of existingFolders) {
try {
const relParts = await newHandle.resolve(existing.handle);
if (relParts && relParts.length > 0) {
return { type: 'parent', child: existing };
}
} catch {}
}
return { type: 'root' };
}
// One request per call: the transaction is the store's own, which is all a
// keyed get/put/delete and a getAll ever need. A cursor-based scan (the folder
// walk) goes through `walk` below instead, because it holds one transaction
// across many requests.
function ask<T>(store: string, mode: IDBTransactionMode, run: (s: IDBObjectStore) => IDBRequest): Promise<T> {
return openDb().then(
(db) =>
new Promise<T>((resolve, reject) => {
const req = run(db.transaction(store, mode).objectStore(store));
req.onsuccess = () => resolve(req.result as T);
req.onerror = () => reject(req.error);
})
);
}
let cachedFoldersList: LibraryFolder[] | null = null;
export function cachedFolders(): LibraryFolder[] {
return cachedFoldersList ?? [];
}
export async function listFolders(): Promise<LibraryFolder[]> {
try {
const rows = await ask<LibraryFolder[]>(FOLDERS, 'readonly', (s) => s.getAll());
const sorted = rows.sort((a, b) => a.name.localeCompare(b.name));
cachedFoldersList = sorted;
return sorted;
} catch {
return cachedFoldersList ?? [];
}
}
export async function reconnectPhotosForFolders(
newFolderHandle: FileSystemDirectoryHandle
): Promise<{ reconnectedFolders: number; reconnectedPhotos: number }> {
const existingFolders = await listFolders();
const allPhotos = (await readPhotos()) ?? [];
let reconnectedFolders = 0;
let reconnectedPhotos = 0;
const folderTargets: { name: string; handle: FileSystemDirectoryHandle }[] = [];
folderTargets.push({ name: newFolderHandle.name, handle: newFolderHandle });
const knownFolderNames = new Set([
...existingFolders.map((f) => f.name),
...allPhotos.map((p) => p.folder),
]);
for (const name of knownFolderNames) {
if (folderTargets.some((t) => t.name === name)) continue;
try {
const sub = await openAt(newFolderHandle, name);
if (sub) {
folderTargets.push({ name, handle: sub });
}
} catch {}
}
const db = await openDb();
await new Promise<void>((resolve, reject) => {
const tx = db.transaction(FOLDERS, 'readwrite');
const store = tx.objectStore(FOLDERS);
for (const target of folderTargets) {
store.put({ name: target.name, handle: target.handle });
reconnectedFolders++;
}
tx.oncomplete = () => resolve();
tx.onerror = () => reject(tx.error);
});
const photosToUpdate: LibraryPhoto[] = [];
for (const target of folderTargets) {
const matchingPhotos = allPhotos.filter((p) => p.folder === target.name && !p.handle);
if (!matchingPhotos.length) continue;
const dirCache = new Map<string, FileSystemDirectoryHandle>();
dirCache.set('', target.handle);
for (const p of matchingPhotos) {
try {
let dirHandle = dirCache.get(p.dir);
if (!dirHandle) {
const opened = await openAt(target.handle, p.dir);
if (!opened) continue;
dirCache.set(p.dir, opened);
dirHandle = opened;
}
const fileHandle = await dirHandle.getFileHandle(p.name);
if (fileHandle) {
p.handle = fileHandle;
photosToUpdate.push(p);
reconnectedPhotos++;
}
} catch {}
}
}
if (photosToUpdate.length) {
await new Promise<void>((resolve, reject) => {
const tx = db.transaction(PHOTOS, 'readwrite');
const store = tx.objectStore(PHOTOS);
for (const p of photosToUpdate) {
store.put(p);
}
tx.oncomplete = () => resolve();
tx.onerror = () => reject(tx.error);
});
}
return { reconnectedFolders, reconnectedPhotos };
}
// The picker's own dialog; `id` makes the browser reopen at the folder this page
// was last pointed at, which is what makes a second visit one click.
export async function pickFolder(): Promise<LibraryFolder | null> {
if (!canBrowseFolders()) throw new Error('no-folder-picker');
const handle = await (
window as unknown as { showDirectoryPicker: (o?: unknown) => Promise<FileSystemDirectoryHandle> }
).showDirectoryPicker({ mode: 'readwrite', id: 'recipescam-library' });
const folder: LibraryFolder = { name: handle.name, handle };
await ask(FOLDERS, 'readwrite', (s) => s.put(folder));
await reconnectPhotosForFolders(handle);
return folder;
}
// A rename is a label over the folder, not a new folder: everything downstream —
// the frame ids, the thumbnail store, the recipe each frame was left at — hangs
// off `name`, which stays the directory's own.
export async function renameFolder(folder: LibraryFolder, label: string): Promise<void> {
await ask(FOLDERS, 'readwrite', (s) => s.put({ ...folder, label }));
}
export async function removeFolder(name: string): Promise<void> {
const ids = (await listPhotos(name)).map((p) => p.id);
const db = await openDb();
await new Promise<void>((resolve, reject) => {
const hasThumbs = db.objectStoreNames.contains(THUMBS);
const stores = hasThumbs ? [FOLDERS, PHOTOS, THUMBS, EDITS] : [FOLDERS, PHOTOS, EDITS];
const tx = db.transaction(stores, 'readwrite');
tx.objectStore(FOLDERS).delete(name);
const photos = tx.objectStore(PHOTOS);
const thumbs = hasThumbs ? tx.objectStore(THUMBS) : null;
const edits = tx.objectStore(EDITS);
for (const id of ids) {
photos.delete(id);
thumbs?.delete(id);
edits.delete(id);
thumbCache.delete(id);
}
tx.oncomplete = () => resolve();
tx.onerror = () => reject(tx.error);
});
// The reading's position goes with the folder: a folder that is gone is not
// one to carry on reading, and a folder picked again starts at the top.
await clearWalk(name);
}
// A handle kept in IndexedDB comes back without its permission: the browser
// dropped it when the tab closed, and only the visitor can hand it back. Asking
// costs one quiet prompt, which is why this runs at start-up rather than the
// moment a frame is clicked. `requestPermission` needs the user's gesture in
// some builds, so a refusal is not fatal — the folder is listed as needing a
// click.
export async function ensurePermission(handle: FileSystemHandle, write = false): Promise<boolean> {
// `FileSystemPermissionMode` is not in the DOM lib this project compiles
// against, and the two values are the whole type.
type PermissionMode = 'read' | 'readwrite';
const mode: PermissionMode = write ? 'readwrite' : 'read';
if (!handle) return false;
const askable = handle as FileSystemHandle & {
queryPermission?: (o: { mode: PermissionMode }) => Promise<PermissionState>;
requestPermission?: (o: { mode: PermissionMode }) => Promise<PermissionState>;
};
try {
if (!askable || !askable.queryPermission) return false;
if ((await askable.queryPermission({ mode })) === 'granted') return true;
return (await askable.requestPermission?.({ mode })) === 'granted';
} catch {
return false;
}
}
// --- thumbnails ------------------------------------------------------------
// EXIF is one JPEG segment, and a segment is 64KB at most — but only a JPEG keeps
// it in the first one, so the slice that is handed to the parser is a few hundred
// kilobytes rather than the whole frame. Reading 8MB through a JS array to find a
// date in the first kilobyte of it is most of what a scan of JPEGs costs.
export const HEAD_BYTES = 256 * 1024;
// The probe is decoded at this width, purely to learn which edge of the frame is
// the long one; the decoder scales the real one off that. A JPEG does not need
// it: its own header says how big the frame is, in the first few hundred bytes.
const PROBE_PX = 8;
// A JPEG's frame header — the SOF segment — carries the frame's size, and it sits
// well inside the bytes already read off the disk for the date. Reading it costs a
// walk over a few hundred bytes where the probe costs a second decode of every
// pixel of the file.
function jpegSize(bytes: Uint8Array): { width: number; height: number } | null {
if (bytes[0] !== 0xff || bytes[1] !== 0xd8) return null;
for (let i = 2; i + 8 < bytes.length; ) {
if (bytes[i] !== 0xff) {
i++;
continue;
}
const marker = bytes[i + 1];
// SOF0–SOF15 minus the ones that are not frames: DHT, JPG, DAC.
if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) {
return { height: (bytes[i + 5] << 8) | bytes[i + 6], width: (bytes[i + 7] << 8) | bytes[i + 8] };
}
const length = (bytes[i + 2] << 8) | bytes[i + 3];
if (length < 2) return null;
i += 2 + length;
}
return null;
}
// What a frame whose header this code cannot read is measured with: one decode
// eight pixels wide. Only the aspect comes out of it — the two numbers that say
// which edge is the long one — and the frame is decoded once more for real.
async function sizeOf(blob: Blob): Promise<{ width: number; height: number } | null> {
try {
const probe = await createImageBitmap(blob, { resizeWidth: PROBE_PX });
const size = { width: probe.width, height: probe.height };
probe.close();
return size;
} catch {
return null;
}
}
// The tile, read at the size the tile is painted at. Asking the decoder for 512
// on the long edge scales on the way out of the decoder — a quarter of the pixels
// of a 24MP frame ever exist — where decoding the frame whole and drawing it
// small pays for every one of them. The probe says which edge that is, and the
// canvas is only here to re-encode to JPEG.
async function tile(blob: Blob, size?: { width: number; height: number } | null): Promise<Blob | null> {
try {
const known = size ?? (await sizeOf(blob));
if (!known) return null;
const wide = known.width >= known.height;
const bitmap = await createImageBitmap(blob, wide ? { resizeWidth: THUMB_MAX } : { resizeHeight: THUMB_MAX });
try {
const canvas = document.createElement('canvas');
canvas.width = bitmap.width;
canvas.height = bitmap.height;
// Opaque: a frame is a rectangle of colour with nothing behind it, and a
// context told so does not carry a channel of noughts through every draw
// and every encode — a quarter less memory on each tile in flight.
const ctx = canvas.getContext('2d', { alpha: false });
if (!ctx) return null;
ctx.drawImage(bitmap, 0, 0);
const res = await new Promise<Blob | null>((resolve) => canvas.toBlob((out) => resolve(out), 'image/jpeg', THUMB_QUALITY));
canvas.width = 0;
canvas.height = 0;
return res;
} finally {
bitmap.close();
}
} catch {
return null;
}
}
// --- scanning --------------------------------------------------------------
export interface ScanProgress {
folder: string;
// The subfolder this reading was kept to, spelled the way the walk spells a
// path (`''` for the whole roll). A reading kept to one folder is a reading of
// a fraction of one roll, and everything downstream — the column's counts, the
// position on the disk — belongs to the reading that walks the whole of it.
from: string;
total: number;
done: number;
added: number;
// How many frames have reached the catalogue. `done` counts the frames the
// reading has got to, which is nearly all of them long before they are stored,
// so a screen that read the catalogue back on `done` would read it again every
// frame — and reading it back means every thumbnail in it again. This moves a
// batch at a time, which is exactly when there is something new to read.
written: number;
// How many frames the scan has read out of each row of the tree, keyed the way
// the tree spells a row (`folder` for the picked folder, `folder/sub/dir` for
// the rest) and counted the way the rows count: a frame sits on its own row and
// on every row above it. A fresh object every frame, so a screen drawing the
// column sees a change; the frames themselves only reach the catalogue in one
// batch at the end, which would leave every row reading zero until then.
counts: Record<string, number>;
// Every folder the walk has been into so far, a fresh list each time: the
// column draws them while the frames under them are still being read. Not
// filed away — a folder name is worth nothing once the scan it came from ends.
dirs: LibraryDir[];
// The folder the reading is standing in at this moment, spelled the way the
// walk spells a path, `''` for the frames that sit in the picked folder
// itself. A reading is where it is and not where it was pointed: a roll is
// walked layer by layer through every folder under it, and a column that
// marked the folder the reading was asked for would mark one row for the
// whole of an afternoon while the work went on somewhere the reader cannot
// see. Where the reading is is a thing only the reading knows, so it is told
// and not worked out.
at: string;
}
// Walk the folder, keep what is new or changed, and leave the rest alone: a
// second scan of a 2000-file folder only reads the frames that moved. A scan
// that was cut off — the tab closed, the reader stopped it — is finished by the
// next one: the frames it got through are skipped on their size and their time,
// so asking again on the way in is the whole of resuming.
//
// The walk itself lives in `rollWalk`: layer by layer from the picked folder
// down, one bounded pass at a time, with the folder the reader just clicked
// pulled to the front of the queue. `jump` is that click.
//
// ponytail: the lanes run on the main thread. The reading itself is done
// elsewhere (disk, the decoder's own threads), so what is left here is small —
// but a 5000-frame folder still makes the page lumpy. Move the walk into a
// worker (LibRaw is happy in one) if a catalogue that large is ever real.
// Where the reading had got to. A reload in the middle of a few thousand frames
// takes the page with it, and what it would take from the reading is the
// position: the folders the walk had been through, the ones still in its queue,
// and the frames it had found and not read. None of that is in the catalogue —
// the catalogue only holds the frames that were read — so the reading would
// start again at the top of the roll and walk every folder a second time.
//
// The position is held for the whole origin, which is the point: the tab that
// reloads carries on from the frame it stopped at, and so does the next context
// to open the app — the installed one beside the browser, where session storage,
// being the tab's own, handed it nothing and the walk started at the top of the
// roll a second time, every folder listed again and every frame it already held
// read again only to be skipped. The file goes when the reading finishes.
//
// It is a file in the origin private file system and not a key in local storage,
// which is where it was. The position is one path per frame the walk has found
// and not read — about twenty-eight characters each, measured over a folder of
// three thousand — and local storage on this origin takes 5,000,000 characters
// and then throws, also measured. That is a hundred and seventy-five thousand
// frames: past it the first write of a long roll fails, the position is nowhere,
// and the next visit walks the whole roll from the top — the reader's own
// complaint, and one a roll of RAW reaches in a morning. OPFS takes the same text
// as a file, has room for a catalogue's worth of them, and is the origin's the
// same way. An unwritable position is still a reading that starts at the top:
// private mode lands there, as it always did.
const WALK_DIR = 'walk';
const WALK_FILE = (folder: string) => `${encodeURIComponent(folder)}.json`;
async function walkFile(folder: string, create: boolean): Promise<FileSystemFileHandle | null> {
const root = await navigator.storage?.getDirectory?.();
if (!root) return null;
const dir = await root.getDirectoryHandle(WALK_DIR, { create });
return dir.getFileHandle(WALK_FILE(folder), { create });
}
interface WalkSaved {
// The folders the walk is through, and the ones it has not read yet, in the
// order it was going to read them.
walked: string[];
pending: string[];
// The frames the walk has found and the reading has not got to. The front of
// this is the frame a reloaded reading picks up at.
frames: string[];
// How many of the front of `frames` were already counted as read by the reading
// that wrote this down: the rows are in hand and not in the catalogue yet, so
// the next reading reads them again — and counts none of them twice.
restored: number;
progress: ScanProgress;
}
async function loadWalk(folder: string): Promise<WalkSaved | null> {
// One reading at a time: a peer that has announced this folder is walking it
// right now, and what it has written down is a step of a walk still in
// progress. Taken up here, two readings would write the same roll down at
// once, each undoing the other's queue. In the tab's own storage this could not
// come up; now that the position is the origin's, it can.
if (peer?.folder === folder) return null;
try {
const fh = await walkFile(folder, false);
if (!fh) return null;
const saved = JSON.parse(await (await fh.getFile()).text()) as WalkSaved;
return Array.isArray(saved?.walked) &&
Array.isArray(saved?.pending) &&
Array.isArray(saved?.frames) &&
Number.isInteger(saved?.restored) &&
saved.progress
? saved
: null;
} catch {
return null;
}
}
// A position that cannot be written down is a reading that starts from the top,
// which is what this did before — private mode lands here, as it always did.
async function saveWalk(
folder: string,
walked: string[],
pending: string[],
frames: string[],
restored: number,
progress: ScanProgress
): Promise<void> {
try {
const fh = await walkFile(folder, true);
if (!fh) return;
const w = await fh.createWritable();
await w.write(JSON.stringify({ walked, pending, frames, restored, progress } satisfies WalkSaved));
await w.close();
} catch {
// Nothing to write down: the reading starts from the top, as it did before.
}
}
async function clearWalk(folder: string): Promise<void> {
try {
const root = await navigator.storage?.getDirectory?.();
const dir = await root?.getDirectoryHandle(WALK_DIR, { create: false });
await dir?.removeEntry(WALK_FILE(folder));
} catch {
// Absent is the desired state.
}
}
// The frames a written-down position names, asked of the root again: a frame is
// spelled by its path, and a handle is not a thing that can be written down. The
// folder part is asked once and kept, because a reading that stopped in a folder
// of a few thousand frames has nearly all of them in the same folder.
async function framesAt(root: FileSystemDirectoryHandle, rels: string[]): Promise<WalkFile[]> {
const dirs = new Map<string, FileSystemDirectoryHandle>();
const out: WalkFile[] = [];
for (const rel of rels) {
const cut = rel.lastIndexOf('/');
const at = cut < 0 ? '' : rel.slice(0, cut);
let dir = dirs.get(at);
if (!dir) {
const opened = await openAt(root, at);
// A folder that will not open takes its frames with it; they are read
// again when the walk reaches it, if it opens at all.
if (!opened) continue;
dirs.set(at, opened);
dir = opened;
}
try {
out.push({ handle: await dir.getFileHandle(rel.slice(cut + 1)), rel });
} catch {
// A frame that is gone is not a frame to read.
}
}
return out;
}
export async function scanFolder(
folder: LibraryFolder,
onProgress?: (p: ScanProgress) => void,
shouldStop?: () => boolean,
jump?: () => string | null,
from = ''
): Promise<ScanProgress> {
const known = new Map((await listPhotos(folder.name)).map((p) => [p.id, p]));
// A reading kept to one folder keeps no position of its own. The position file
// belongs to the reading that walks the whole roll, and a fraction of the tree
// written into it would hand the next visit a roll with the rest of itself
// missing from the walk. One folder is short enough to read again, and the
// frames it does not re-read are skipped on their size and their time.
const bounded = from !== '';
const saved = bounded ? null : await loadWalk(folder.name);
const treeWalked = new Set(saved?.walked ?? []);
if (from) {
for (const p of [...treeWalked]) {
if (p === from || p.startsWith(from)) treeWalked.delete(p);
}
}
const tree: Walk = { root: folder.handle, pending: [], walked: treeWalked };
// A queue comes back as paths, so the folders are asked for again; a reading
// with nothing written down starts at the folder it was pointed at — the
// picked one, or the one the reader right-clicked inside it.
if (saved) {
for (const rel of saved.pending) {
const dir = await openAt(folder.handle, rel);
if (dir) tree.pending.push({ dir, rel });
}
if (from && !tree.pending.some((p) => p.rel === from)) {
const dir = await openAt(folder.handle, from);
if (dir) tree.pending.unshift({ dir, rel: from });
}
} else {
const dir = from ? await openAt(folder.handle, from) : folder.handle;
if (dir) tree.pending.push({ dir, rel: from });
}
// The frames this reading has found and not read: the walk fills it, the lanes
// empty it, and a frame taken off the front is one a reload will not read
// again. Also where a reading that was cut off picks up.
const entries: WalkFile[] = saved ? await framesAt(folder.handle, saved.frames) : [];
// What the passes found: the folders this scan walked into, and the names they
// put in the column. `names` is cleared a pass at a time; `found` keeps the
// whole scan so a folder that has gone from the disk goes from the column too.
const names: string[] = [];
const progress: ScanProgress = saved?.progress ?? {
folder: folder.name,
from,
total: 0,
done: 0,
added: 0,
written: 0,
dirs: [],
counts: {},
at: '',
};
progress.folder = folder.name;
// Which folder the reading was pointed at, for the same reason the name is:
// the screen up when the reading ends is not the one that started it, and a
// reading kept to one folder has to say so to a column that would otherwise
// draw its counts over the whole roll.
progress.from = from;
// A position written down by an older visit has no count to carry on from, and
// no folder to say it is in until the first frame is in hand again.
progress.written ??= 0;
progress.at ??= '';
const found = new Set(progress.dirs.map((d) => d.id));
const counts: Record<string, number> = progress.counts;
// Count the frame the moment the scan gets to it, before it knows whether the
// frame is new: the rows say how far the reading has come, not what it kept.
const count = (rel: string) => {
const cut = rel.lastIndexOf('/');
counts[folder.name] = (counts[folder.name] ?? 0) + 1;
let path = '';
for (const part of (cut < 0 ? '' : rel.slice(0, cut)).split('/')) {
if (!part) continue;
path = path ? `${path}/${part}` : part;
const key = `${folder.name}/${path}`;
counts[key] = (counts[key] ?? 0) + 1;
}
};
// The frames the catalogue already holds of the folder being read, which is
// what the reading is measured against: a roll of a hundred thousand that has
// been read once before is a reading of a hundred thousand frames even while
// the walk is still up in its first folders, and one kept to a subfolder is
// measured against the frames under that one alone.
const held = bounded
? [...known.keys()].filter((id) => id.startsWith(`${folder.name}/${from}`)).length
: known.size;
// What the screen is told, and how often. The column is redrawn out of the
// count and the count moves with every frame, so telling the screen a frame at
// a time is a whole tree rebuilt a frame at a time — on a roll of a hundred
// thousand that is the reading spending its afternoon drawing itself, and the
// reason a scan looks like it has started over from the top. Every frame still
// lands in `counts`; what waits is the copy the screen reads, and the count of
// frames reached.
let toldAt = 0;
const publish = (force = false) => {
const at = Date.now();
if (!force && at - toldAt < TELL_MS) return;
toldAt = at;
progress.counts = { ...counts };
// Never backwards, and never less than the frames the catalogue holds: the
// walk finds the rest of them as it goes, and a line reading `29980/61211`
// over a roll of a hundred thousand is the counter talking, not the roll.
progress.total = Math.max(progress.total, progress.done + entries.length, held);
onProgress?.(progress);
};
// The folders this scan has walked into, kept for the whole of it: the column
// is redrawn from this list, so it only ever grows. A reading that was cut off
// hands the list on with the rest of its position.
const dirs: LibraryDir[] = progress.dirs;
let batch: LibraryPhoto[] = [];
// The frames this reading found for the first time, waiting to be handed on
// with the batch they landed in. A frame the catalogue already had is not one
// of these, however it came back — it is on the strip already, and a second
// copy of it is a second row under one key.
let added: LibraryPhoto[] = [];
// The frames in hand: the ones a lane has been handed, and the ones whose row is
// in `batch` and not yet in the catalogue. They are on neither side of the line a
// position is written on — the catalogue does not hold them and the walk will not
// find them again — so a reload has to read them again, and they are also the
// frames the written-down count has already counted.
let inHand: string[] = [];
const pathOf = (row: LibraryPhoto) => (row.dir ? `${row.dir}/${row.name}` : row.name);
// The position goes down on a clock, not on every batch. The whole of it is
// written each time — a copy of every path the walk has found and not read —
// and on a roll of a hundred thousand that is a few megabytes turned into a
// string per batch, which is the page held still for the whole roll. A position
// a step behind is a frame or two read again on the next visit, and a frame read
// again is skipped on its size and its time.
let wroteAt = 0;
const write = async () => {
// A reading kept to one folder has no position to write: the file it would
// go in is the whole roll's, and a fraction of the tree written there is a
// roll that comes back with the rest of itself missing.
if (bounded) return;
const at = Date.now();
if (at - wroteAt < WALK_MS) return;
wroteAt = at;
const held = [...inHand, ...batch.map(pathOf)];
const savedEntries = entries.slice(0, 500).map((e) => e.rel);
await saveWalk(
folder.name,
[...tree.walked],
tree.pending.map((p) => p.rel),
[...held, ...savedEntries],
held.length,
progress
);
};
// One transaction per batch, a put per row: a store with `keyPath: 'id'` takes
// a record, not an array of them. The position is written with it — a batch is
// the beat the reading and the stored position are kept in step at, so at most
// one batch is read again after a reload, and a frame that is read again is
// skipped on its size and its time.
const flush = async () => {
if (!batch.length) return;
const rows = batch;
batch = [];
const db = await openDb();
await new Promise<void>((resolve, reject) => {
const hasThumbs = db.objectStoreNames.contains(THUMBS);
const stores = hasThumbs ? [PHOTOS, THUMBS] : [PHOTOS];
const tx = db.transaction(stores, 'readwrite');
const photoStore = tx.objectStore(PHOTOS);
const thumbStore = hasThumbs ? tx.objectStore(THUMBS) : null;
for (const row of rows) {
const { thumb, ...meta } = row;
photoStore.put(meta);
if (thumb) {
cacheThumb(row.id, thumb);
thumbStore?.put({ id: row.id, thumb });
}
}
tx.oncomplete = () => resolve();
tx.onerror = () => reject(tx.error);
});
progress.written += rows.length;
if (added.length) {
const fresh = added;
added = [];
rowSink?.(fresh);
}
await write();
};
const readOne = async (handle: FileSystemFileHandle, rel: string): Promise<void> => {
const file = await handle.getFile();
const id = photoId(folder.name, rel);
const seen = known.get(id);
if (seen && seen.size === file.size && seen.mtime === file.lastModified) {
if (!seen.handle) {
batch.push({ ...seen, handle });
if (batch.length >= BATCH) await flush();
}
return;
}
const raw = isRawName(file.name);
const isTiff = /\.(tiff?)$/i.test(file.name);
const heic = isHeicName(file.name);
// Slice buffer: HEIC uses up to 4MB; RAW & TIFF use up to 32MB (or full file if smaller).
const MAX_SLICE_BYTES = (raw || isTiff) ? 32 * 1024 * 1024 : heic ? 4 * 1024 * 1024 : HEAD_BYTES;
const sliceBuffer = await file.slice(0, Math.min(file.size, MAX_SLICE_BYTES)).arrayBuffer();
const bytes = new Uint8Array(sliceBuffer);
let preview: Uint8Array | null = null;
if (heic) {
preview = await heicThumbnail(bytes);
if (!preview) {
try {
const fullBytes = new Uint8Array(await file.arrayBuffer());
preview = await heicToJpeg(fullBytes, 320);
} catch {
preview = null;
}
}
} else if (raw || isTiff) {
preview = await rawThumbnail(bytes, file.name);
if (!preview && isTiff) {
try {
const fullBytes = new Uint8Array(await file.arrayBuffer());
preview = tiffThumbnail(fullBytes);
} catch {
preview = null;
}
}
}
const src = preview ? new Blob([preview as BlobPart], { type: 'image/jpeg' }) : (raw || heic || isTiff) ? null : file;
const size = preview ? jpegSize(preview) : (raw || heic || isTiff) ? null : jpegSize(bytes);
const thumb = src ? await tile(src, size) : null;
const taken = (await readCapturedAt(bytes)) ?? file.lastModified;
const relNorm = rel.replace(/\\/g, '/');
const cut = relNorm.lastIndexOf('/');
const row: LibraryPhoto = {
id,
folder: normPath(folder.name),
dir: cut < 0 ? '' : normPath(relNorm.slice(0, cut)),
name: handle.name,
handle,
thumb,
taken,
size: file.size,
mtime: file.lastModified,
star: seen?.star,
rot: seen?.rot,
addedAt: seen?.addedAt ?? Date.now(),
};
batch.push(row);
if (!seen) added.push(row);
progress.added++;
inHand = inHand.filter((r) => r !== rel);
if (batch.length >= BATCH) await flush();
};
// The frames in hand, read a few at a time: reading a frame is almost all
// waiting — the bytes come off the disk, the decode runs on a thread of its own
// — and the wait of one frame is spent on the next one instead of sitting
// still. Lanes are why the same roll lands in a fraction of the time; past a
// few the disk and the decoder are the limit. A frame leaves the queue the
// moment its lane is handed it, so the position on disk trails the reading by
// no more than one batch.
const drain = async () => {
const lanes = new Set<Promise<void>>();
let processed = 0;
while (entries.length) {
if (shouldStop?.()) {
stop = true;
break;
}
const { handle, rel } = entries.shift()!;
inHand.push(rel);
// The frame in hand names the folder the reading is standing in, and the
// queue is filled layer by layer, so this is the column's answer to which
// folder is being read while it is being read.
const at = rel.lastIndexOf('/');
progress.at = at < 0 ? '' : rel.slice(0, at);
// A frame a reload handed back that was counted already is read without
// being counted twice; the ones behind it are frames that reading found and
// never reached, and they are counted here as they are reached.
if (restored) restored--;
else {
progress.done++;
count(rel);
}
// Queue image decoding one at a time so memory usage stays low and UI stays smooth
const read = oneRawAtATime(() => readOne(handle, rel));
const lane = read
.catch(() => undefined)
.finally(() => lanes.delete(lane));
lanes.add(lane);
if (lanes.size >= LANES) await Promise.race(lanes);
publish();
// Micro-yield to the browser main thread event loop so scanning never blocks UI rendering or user interactions
processed++;
if (processed % 3 === 0) {
await new Promise((resolve) => setTimeout(resolve, 0));
}
}
await Promise.all(lanes);
await flush();
};
// How many of the frames in hand were counted by the reading that handed them
// back: the rows it had read and not yet stored. The rest of the queue is frames
// it found and never got to, and this reading counts those as it reaches them —
// the rows and the counter both say how far the reading has come, and a reload
// that stopped at a frame must not count it twice.
let restored = Math.min(saved?.restored ?? 0, entries.length);
let stop = false;
let walked = false;
// One bounded pass at a time, and the pass yields between frames, so a folder
// clicked while the scan runs is read on the next jump rather than after the
// whole tree. Nothing clicked: the picked folder and its frames, then each
// layer below with theirs, deepest last — the order `pending` is filled in.
do {
names.length = 0;
// The queue a reading that was cut off left behind goes first, and the walk
// is asked for nothing until it is through: the frames already found are
// what that reading was in the middle of.
if (!entries.length) {
// A reading kept to one folder is asked for nothing: the frames of the
// folder it was pointed at, and of what lies below it, and then it is
// through. A jump would take it out of the subtree it was kept to, and
// the reading it left behind is the reading that folder wanted.
const asked = bounded ? () => null : (jump ?? (() => null));
walked = await walkPass(tree, entries, names, isSupportedPhoto, asked);
// The names come in before the frames they hold: the column grows one pass
// ahead of the strip, which is the whole point of reading layer by layer.
for (const rel of names) {
const id = `${folder.name}/${rel.slice(0, -1)}`;
if (found.has(id)) continue;
found.add(id);
dirs.push({ id, folder: folder.name, rel: rel.slice(0, -1) });
}
// The column is handed the names the moment the pass is through, before a
// single frame under them has been read. A copy, so the redraw has
// something new to look at rather than the list growing under it, and a
// reading that is cut off hands the same list on with its position.
progress.dirs = dirs.slice();
// The position before a frame of this pass is looked at: a reload here
// reads the pass's frames again, and a frame already in the catalogue is
// skipped anyway — where the other order would leave the last pass's
// frames out of the catalogue until the next visit.
await write();
// A pass is worth telling the screen about whether the clock says so or
// not: the names of a whole layer have just arrived, and the column is
// built from them.
publish(true);
}
await drain();
// Yield between pass iterations to maintain UI fluidness
await new Promise((resolve) => setTimeout(resolve, 0));
} while (!walked && !stop);
await flush();
// A reading that came to its end has no position worth keeping: the next one
// walks the roll from the top and skips what has not moved. A reading kept to
// one folder never wrote one down, and the file it would clear is the whole
// roll's — a position another reading is in the middle of.
if (!stop && !bounded) await clearWalk(folder.name);
// The last word, whatever the clock says: the frames the reading got to are
// the frames the column is about to be told about one last time.
publish(true);
return progress;
}
// --- the scan in flight ----------------------------------------------------
// A scan belongs to the tab, not to the screen it was started from. A roll takes
// minutes, and the reader who started it has every reason to go to the studio and
// work on a frame while it runs — so the screen that started it may well be gone
// before the scan is. What that screen would have held lives here instead: one
// scan at a time, watched by whichever screen is up.
export interface ScanSession {
// The folder being read, by name.
folder: string;
progress: ScanProgress;
// What the reader has asked for since: stop, or read this folder next (the path
// as `rollWalk` spells it — `''` for the picked folder itself).
stop: boolean;
jump: string | null;
}
let live: ScanSession | null = null;
const watchers = new Set<() => void>();
const tell = () => {
for (const fn of watchers) fn();
};
// --- the other windows on this origin --------------------------------------
// An installed app and a browser tab share one origin, one catalogue and one
// roll: without a word between them the second window opens, finds nothing in
// flight here, and reads the same frames again — two readers of one folder, two
// RAW decoders at 256MB apiece, for a progress the first window already has.
// So the window holding the reading says so, on every frame and on a heartbeat,
// and the windows that do not hold it take the reading as their own: they draw
// the same progress, offer the same stop, and above all start nothing.
const CHANNEL = 'recipescam-library';
const SELF = Math.random().toString(36).slice(2);
// A window that has heard nothing for this long has a peer that was closed, or
// one that is wedged — either way the reading is nobody's again, and this window
// may pick it up as it always could.
const PEER_MS = 6000;
const HEART_MS = 2000;
// One round trip, for the window that has just opened: whether another window
// holds the reading is what decides whether this one starts its own.
const ASK_MS = 250;
interface Peer {
folder: string;
progress: ScanProgress;
at: number;
}
let peer: Peer | null = null;
let peerTimer: ReturnType<typeof setTimeout> | null = null;
let heart: ReturnType<typeof setInterval> | null = null;
let askers: ((busy: boolean) => void)[] = [];
// A browser without the channel (an old one, a worker) simply has no peers, and
// everything below is a no-op there.
const channel = typeof BroadcastChannel === 'function' ? new BroadcastChannel(CHANNEL) : null;
const say = (m: Record<string, unknown>): void => {
channel?.postMessage({ ...m, from: SELF });
};
const dropPeer = (): void => {
if (peerTimer) {
clearTimeout(peerTimer);
peerTimer = null;
}
if (!peer) return;
peer = null;
tell();
};
const holdPeer = (folder: string, progress: ScanProgress): void => {
peer = { folder, progress, at: Date.now() };
for (const fn of askers) fn(true);
askers = [];
if (peerTimer) clearTimeout(peerTimer);
peerTimer = setTimeout(dropPeer, PEER_MS);
tell();
};
// A peer holds the reading the way this window holds one, as far as a screen is
// concerned: the same shape, read the same way. Whether the reading is stopped
// or jumped from here is answered by relaying it, which is why the window
// watching can offer the buttons the window reading has.
const peerSession = (): ScanSession | null =>
peer ? { folder: peer.folder, progress: peer.progress, stop: false, jump: null } : null;
const announce = (): void => {
if (live) say({ k: 'scan', folder: live.folder, progress: live.progress });
};
if (channel) {
channel.onmessage = (e: MessageEvent) => {
const m = e.data as { k?: string; from?: string; folder?: string; progress?: ScanProgress; rel?: string | null } | null;
if (!m || m.from === SELF) return;
if (m.k === 'scan' && m.folder && m.progress) holdPeer(m.folder, m.progress);
else if (m.k === 'end') {
if (peer?.folder === m.folder) dropPeer();
} else if (m.k === 'who') announce(); // a window just opened and asks who reads
else if (m.k === 'stop' && live && live.folder === m.folder) {
live.stop = true;
tell();
} else if (m.k === 'jump' && live && live.folder === m.folder) live.jump = m.rel ?? null;
};
}
// Whether another window is reading right now, asked rather than assumed: the
// window that opens beside a running scan knows nothing until the reading
// answers, and reading the disk twice over is exactly what the answer prevents.
export function scanBusy(): Promise<boolean> {
if (live) return Promise.resolve(true);
if (!channel) return Promise.resolve(false);
const answered = new Promise<boolean>((resolve) => {
const settle = (busy: boolean): void => {
askers = askers.filter((fn) => fn !== settle);
resolve(busy);
};
askers.push(settle);
// Nobody answered: no reading is running, and waiting longer only delays
// this window's own.
setTimeout(() => settle(false), ASK_MS);
});
say({ k: 'who' });
return answered;
}
// The scan in flight, or null. The same object for the whole of the scan, with
// `progress` replaced on every pass — watchers are told, so they re-read it.
// A reading another window holds answers the same way: the screens no longer
// know or care which window the reading is in.
export function scanSession(): ScanSession | null {
return live ?? peerSession();
}
// Watch, and get the unsubscribe back: the shape an effect already has.
export function watchScan(fn: () => void): () => void {
watchers.add(fn);
return () => {
watchers.delete(fn);
};
}
// The rows a reading has just stored, handed to the screen that is up as they
// land. A scan of a hundred thousand frames stores a batch a second, and the
// strip that waited to be told by a read of the whole catalogue spent the roll
// doing nothing else — a second and a half of the page held still, sixty
// megabytes of rows thrown away, every few seconds, while beside it a RAW
// decoder held a quarter of a gigabyte. The window that holds the reading is the
// one that hears this; a window watching from the side reads the catalogue back
// on its clock, as it always did.
let rowSink: ((rows: LibraryPhoto[]) => void) | null = null;
export function watchRows(fn: ((rows: LibraryPhoto[]) => void) | null): void {
rowSink = fn;
}
// Whether a reading of this folder was left half-done. A position on the disk is
// a roll that was cut off before its end and the only thing that finishes it is
// reading it again; a peer holding the same folder is the same answer, from the
// window that is already doing it. A question and not a walk: one small file is
// opened and shut, and not a name on the disk is looked at.
export async function unfinished(folder: string): Promise<boolean> {
if (peer?.folder === folder) return true;
return (await loadWalk(folder)) !== null;
}
export function stopScan(): void {
// Everything waiting is a reading that is not going to happen: the button that
// stops the walk stops the afternoon, and a queue that ran on after it would be
// a stop that stopped one folder and then read the next three anyway.
waiting.forEach(drop);
waiting = [];
tell();
// A reading another window holds is stopped by telling it so: the button this
// window drew is the same button, on the window that can act on it.
if (!live) {
if (peer) say({ k: 'stop', folder: peer.folder });
return;
}
live.stop = true;
tell();
}
// The row's own stop: the request for this row that is still waiting its turn,
// and the reading of this roll, which is one at a time — a row under the roll
// stops the reading the same way the roll's own row does, because the reading it
// stops is the roll's. The other rolls' requests are other readings and run on.
export function stopScanAt(folder: string, rel: string): void {
for (const job of waiting) {
// The roll's own row stops the whole roll — the requests under it are readings
// of that one roll, a folder at a time. A subfolder's row stops that folder.
if (job.folder === folder && (rel === '' || job.rel === rel)) drop(job);
}
waiting = waiting.filter((j) => !j.dropped);
if (live?.folder === folder) live.stop = true;
else if (peer?.folder === folder) say({ k: 'stop', folder });
tell();
}
export function jumpScan(rel: string | null): void {
if (live) {
live.jump = rel;
return;
}
if (peer) say({ k: 'jump', folder: peer.folder, rel });
}
// Start reading a roll. Only one at a time: the walk reads one frame at a time on
// this thread, so a second scan would only slow the first one down. The promise
// settles when the scan does — the caller that started it may be long gone.
//
// `update` is the reading that picks up where the last one left off: the position
// on the disk if there is one, the whole roll from the top if there is not, with
// every frame the catalogue already holds skipped on its size and its time.
// `rescan` is the same reading refusing that position — the roll walked again
// from the top — which is what finds a folder that was added while a reading was
// cut off, and what a frame written over is found by.
export function startScan(folder: LibraryFolder, from = '', mode: ScanMode = 'update'): Promise<ScanProgress> {
if (live) return Promise.reject(new Error('a scan is already running'));
// A reading kept to one folder under the roll keeps no position of its own and
// reads that folder however it was asked for, so only a roll's own rescan has a
// position to throw away — and throwing away a position this reading is not
// going to use would cost the next update the rest of the roll.
const go = mode === 'rescan' && !from ? clearWalk(folder.name) : Promise.resolve();
const session: ScanSession = {
folder: folder.name,
progress: { folder: folder.name, from, total: 0, done: 0, added: 0, written: 0, dirs: [], counts: {}, at: '' },
stop: false,
jump: null,
};
live = session;
tell();
announce();
// Between two frames there is nothing to announce, and a window that hears
// silence long enough would take the reading for abandoned and start its own.
if (heart) clearInterval(heart);
heart = setInterval(announce, HEART_MS);
return go.then(() =>
scanFolder(
folder,
(p) => {
session.progress = p;
tell();
announce();
},
() => session.stop,
() => session.jump,
from
)
).finally(() => {
if (heart) {
clearInterval(heart);
heart = null;
}
live = null;
say({ k: 'end', folder: session.folder });
tell();
});
}
// --- what the reader asked for, in the order they asked ---------------------
// One reading at a time is not one request at a time: a reader who right-clicks
// four folders means four readings, and the walk is one thread — a second reading
// of the same disk only slows the first. So the ones that cannot run yet wait
// here, in the order they were asked for. The folder asked for while nothing runs
// starts at once; the one asked for during a reading starts when that reading
// ends, whether it ran to its end or was stopped; and a request taken out of the
// queue is a reading that never happens.
export type ScanMode = 'update' | 'rescan';
export interface ScanJob {
// The picked folder the reading belongs to, by name.
folder: string;
// The row it was asked of, spelled the way the walk spells a path: '' for the
// picked folder itself, `2026/04/` for a folder under it.
rel: string;
mode: ScanMode;
}
interface Waiting extends ScanJob {
// The picker's own handle, which is what a reading is started from: a name is
// not a folder that can be walked.
handle: FileSystemDirectoryHandle;
dropped: boolean;
settle: (p: ScanProgress | null) => void;
fail: (err: unknown) => void;
// The same request asked twice is one reading, and this is the answer both
// askers are waiting for.
answer: Promise<ScanProgress | null>;
}
let waiting: Waiting[] = [];
const drop = (job: Waiting): void => {
if (job.dropped) return;
job.dropped = true;
job.settle(null);
};
// What is waiting its turn, in the order it will be read: what a row draws its
// mark from, and the whole of what a screen knows about the queue.
export function scanQueue(): ScanJob[] {
return waiting.filter((j) => !j.dropped).map(({ folder, rel, mode }) => ({ folder, rel, mode }));
}
// Ask for a reading of one folder: the row that was right-clicked, and whether
// what is wanted is what is new under it or the whole of it again. A request is
// never refused and never starts a second reading — it waits its turn and answers
// with the reading it asked for, or with null when it was taken out of the queue
// before it ran.
export function scanAsked(folder: LibraryFolder, rel = '', mode: ScanMode = 'update'): Promise<ScanProgress | null> {
const same = waiting.find((j) => !j.dropped && j.folder === folder.name && j.rel === rel && j.mode === mode);
if (same) return same.answer;
let settle!: (p: ScanProgress | null) => void;
let fail!: (err: unknown) => void;
const answer = new Promise<ScanProgress | null>((resolve, reject) => {
settle = resolve;
fail = reject;
});
waiting.push({ folder: folder.name, rel, mode, handle: folder.handle, dropped: false, settle, fail, answer });
tell();
pump();
return answer;
}
// Hand the next request its turn. The reading that just ended is the one thing
// that can make room for it, and the window that holds the reading is the one
// that hears when it ends.
function pump(): void {
if (live) return;
const job = waiting.find((j) => !j.dropped);
if (!job) return;
// A reading another window holds is waited out rather than raced: the disk is
// one disk, and two readers of one roll is the whole of what this is for. The
// turn is taken when that window says it is through, which is the word the
// screens draw their progress from.
if (peer) {
const stop = watchScan(() => {
if (peer) return;
stop();
pump();
});
return;
}
waiting = waiting.filter((j) => j !== job);
startScan({ name: job.folder, handle: job.handle }, job.rel, job.mode).then(job.settle, job.fail).finally(pump);
}
// --- reading ---------------------------------------------------------------
// The catalogue, read once and kept. STUDIO and LIBRARY are two screens of one
// page, and the reader who goes to the studio to develop a frame and comes back
// is not a reader who wants a hundred thousand rows read and sorted again to see
// the strip they were just looking at. The read is the one `listPhotos` always
// did; what it found is left here, and the screen that opens next draws from it
// before the read it just started has answered.
let cached: LibraryPhoto[] | null = null;
export function cachedPhotos(): LibraryPhoto[] {
return cached ?? [];
}
export function clearLibraryCache(): void {
cached = null;
}
let backupTileFetcher: ((id: string) => Promise<Blob | null>) | null = null;
export function setBackupTileFetcher(fetcher: ((id: string) => Promise<Blob | null>) | null): void {
backupTileFetcher = fetcher;
}
// Null, and not an empty list: a read that failed is not a catalogue with
// nothing in it, and the screen that would draw an empty strip on one is the
// screen that loses the frames it was showing — under a scan of its own, where
// the read is the thing that gives way first, that is every time.
export async function getPhotoThumb(id: string): Promise<Blob | null> {
const cached = getCachedThumb(id);
if (cached) return cached;
try {
const db = await openDb();
if (db.objectStoreNames.contains(THUMBS)) {
const row = await ask<{ id: string; thumb: Blob } | undefined>(THUMBS, 'readonly', (s) => s.get(id));
if (row?.thumb) {
cacheThumb(id, row.thumb);
return row.thumb;
}
}
} catch {}
if (backupTileFetcher) {
try {
const blob = await backupTileFetcher(id);
if (blob) {
cacheThumb(id, blob);
try {
const db = await openDb();
if (db.objectStoreNames.contains(THUMBS)) {
const tx = db.transaction(THUMBS, 'readwrite');
tx.objectStore(THUMBS).put({ id, thumb: blob });
}
} catch {}
return blob;
}
} catch {}
}
return null;
}
export async function getPhotoThumbs(ids: string[]): Promise<Map<string, Blob>> {
const res = new Map<string, Blob>();
const missing: string[] = [];
for (const id of ids) {
const cached = getCachedThumb(id);
if (cached) {
res.set(id, cached);
} else {
missing.push(id);
}
}
if (!missing.length) return res;
const stillMissing: string[] = [];
try {
const db = await openDb();
if (db.objectStoreNames.contains(THUMBS)) {
await new Promise<void>((resolve, reject) => {
const tx = db.transaction(THUMBS, 'readonly');
const store = tx.objectStore(THUMBS);
for (const id of missing) {
const req = store.get(id);
req.onsuccess = () => {
const row = req.result as { id: string; thumb: Blob } | undefined;
if (row?.thumb) {
cacheThumb(id, row.thumb);
res.set(id, row.thumb);
} else {
stillMissing.push(id);
}
};
}
tx.oncomplete = () => resolve();
tx.onerror = () => reject(tx.error);
});
} else {
stillMissing.push(...missing);
}
} catch {
stillMissing.push(...missing);
}
if (stillMissing.length && backupTileFetcher) {
const fetcher = backupTileFetcher;
await Promise.all(
stillMissing.map(async (id) => {
try {
const blob = await fetcher(id);
if (blob) {
cacheThumb(id, blob);
res.set(id, blob);
try {
const db = await openDb();
if (db.objectStoreNames.contains(THUMBS)) {
const tx = db.transaction(THUMBS, 'readwrite');
tx.objectStore(THUMBS).put({ id, thumb: blob });
}
} catch {}
}
} catch {}
})
);
}
return res;
}
export async function readPhotos(folder?: string): Promise<LibraryPhoto[] | null> {
try {
const db = await openDb();
const rows = await ask<LibraryPhoto[]>(PHOTOS, 'readonly', (s) =>
folder ? s.index('folder').getAll(folder) : s.getAll()
);
for (let i = 0; i < rows.length; i++) {
if (rows[i].thumb) delete (rows[i] as { thumb?: Blob | null }).thumb;
}
rows.sort((a, b) => b.taken - a.taken);
if (!folder) cached = rows;
return rows;
} catch {
return null;
}
}
export async function listPhotos(folder?: string): Promise<LibraryPhoto[]> {
return (await readPhotos(folder)) ?? [];
}
export async function getPhoto(id: string): Promise<LibraryPhoto | null> {
try {
const photo = (await ask<LibraryPhoto | undefined>(PHOTOS, 'readonly', (s) => s.get(id))) ?? null;
if (photo && photo.thumb) {
cacheThumb(id, photo.thumb);
delete (photo as { thumb?: Blob | null }).thumb;
}
return photo;
} catch {
return null;
}
}
// Score a frame, or take the score back with a 0. Nothing on the disk changes —
// the stars are the catalogue's own — so this is a read and a put, and a frame
// that is read again on the next scan carries its score over.
export async function setStar(id: string, star: number): Promise<void> {
const photo = await getPhoto(id);
if (!photo) return;
try {
await ask(PHOTOS, 'readwrite', (s) => s.put({ ...photo, star }));
} catch {
// Private mode: the score lives no longer than the visit.
}
}
// The quarter turn a frame stands at, in the same read-and-put the score uses:
// the catalogue's own, carried over by a rescan, and what the studio opens the
// frame at.
export async function setRotation(id: string, rot: 0 | 90 | 180 | 270): Promise<void> {
const photo = await getPhoto(id);
if (!photo) return;
try {
await ask(PHOTOS, 'readwrite', (s) => s.put({ ...photo, rot }));
} catch {
// Private mode: the turn lives no longer than the visit.
}
}
// The frame itself, at last: nothing is read from the disk until this runs, so
// the catalogue costs thumbnails and no more.
export async function readPhotoFile(photo: LibraryPhoto): Promise<File> {
return photo.handle.getFile();
}
// --- edits -----------------------------------------------------------------
export async function saveEdit(id: string, recipe: Recipe): Promise<void> {
try {
const edit: LibraryEdit = { photoId: id, recipe, updatedAt: Date.now() };
await ask(EDITS, 'readwrite', (s) => s.put(edit));
} catch {
// Private mode: the frame is still editable, it just forgets the recipe.
}
}
export async function loadEdit(id: string): Promise<LibraryEdit | null> {
try {
return (await ask<LibraryEdit | undefined>(EDITS, 'readonly', (s) => s.get(id))) ?? null;
} catch {
return null;
}
}
export async function listEditedIds(): Promise<Set<string>> {
try {
const rows = await ask<LibraryEdit[]>(EDITS, 'readonly', (s) => s.getAll());
return new Set(rows.map((e) => e.photoId));
} catch {
return new Set();
}
}
export async function listEdits(): Promise<LibraryEdit[]> {
try {
return await ask<LibraryEdit[]>(EDITS, 'readonly', (s) => s.getAll());
} catch {
return [];
}
}
// --- restore ---------------------------------------------------------------
// The catalogue as it travels: the rows without the two things that cannot be
// written to a file — a handle, which only the visitor's picker can hand out, and
// the tile itself, which is a blob and would be a second copy of the backup
// folder. Everything else is here, `mtime` and `size` and `star` included, which
// is what lets the walk that follows match every frame and read none of them.
export type CatalogueRow = Omit<LibraryPhoto, 'handle' | 'thumb'>;
// The rows a set of ids stands for, gaps left as gaps — one transaction for the
// lot, where `getPhoto` would be one per frame.
async function rowsAt(ids: string[]): Promise<(LibraryPhoto | undefined)[]> {
const db = await openDb();
return new Promise((resolve, reject) => {
const tx = db.transaction(PHOTOS, 'readonly');
const store = tx.objectStore(PHOTOS);
const out: (LibraryPhoto | undefined)[] = [];
ids.forEach((id, i) => {
const req = store.get(id);
req.onsuccess = () => {
out[i] = req.result as LibraryPhoto | undefined;
};
});
tx.oncomplete = () => resolve(out);
tx.onerror = () => reject(tx.error);
});
}
// Put a catalogue back. A frame the catalogue already holds and can open — one
// with a handle — is left exactly as it is: the backup's copy of it would come
// back without one, which is a working catalogue traded for a worse one. Every
// other row goes in as it came out. `thumb` fetches the tile that was backed up
// beside the row, and a tile that will not read leaves the row without one: the
// grid paints a blank cell until the next scan reads the frame.
export async function restoreCatalogue(
photos: CatalogueRow[],
edits: LibraryEdit[],
thumb?: (id: string) => Promise<Blob | null>
): Promise<number> {
const db = await openDb();
const size = 200;
let put = 0;
const hasThumbs = db.objectStoreNames.contains(THUMBS);
for (let at = 0; at < photos.length; at += size) {
const slice = photos.slice(at, at + size);
const had = await rowsAt(slice.map((r) => r.id));
const want = slice.filter((_, i) => !had[i]?.handle);
if (!want.length) continue;
const items = await Promise.all(
want.map(async (row) => {
const blob = thumb ? await thumb(row.id) : null;
return { row, blob };
})
);
await new Promise<void>((resolve, reject) => {
const stores = hasThumbs ? [PHOTOS, THUMBS] : [PHOTOS];
const tx = db.transaction(stores, 'readwrite');
const photoStore = tx.objectStore(PHOTOS);
const thumbStore = hasThumbs ? tx.objectStore(THUMBS) : null;
for (const { row, blob } of items) {
const { thumb: _, ...meta } = row as any;
photoStore.put(meta);
if (blob) {
cacheThumb(row.id, blob);
thumbStore?.put({ id: row.id, thumb: blob });
}
}
tx.oncomplete = () => resolve();
tx.onerror = () => reject(tx.error);
});
put += items.length;
}
for (let at = 0; at < edits.length; at += size) {
const slice = edits.slice(at, at + size);
await new Promise<void>((resolve, reject) => {
const tx = db.transaction(EDITS, 'readwrite');
const store = tx.objectStore(EDITS);
for (const row of slice) store.put(row);
tx.oncomplete = () => resolve();
tx.onerror = () => reject(tx.error);
});
}
return put;
}