library: the reading is copied into a folder of the reader's own, so a cleared profile gets it back without a frame being read twice

2026-09-30 16:41:11 +07:00
parent 325d5c0470
commit 3e3045d912
6 changed files with 902 additions and 1 deletions
@@ -572,7 +572,17 @@ export async function scanFolder(
    // exactly as the catalogue already holds it. This is what makes a second scan
    // of the same folder cheap, and what finishes a roll whose first scan was cut
    // short — the frames it got through are skipped, the rest are read.
-    if (seen && seen.size === file.size && seen.mtime === file.lastModified && seen.thumb) return;
+    if (seen && seen.size === file.size && seen.mtime === file.lastModified && seen.thumb) {
+      // A row restored from a backup has no handle: a handle is not a thing that
+      // can be written to a file, so the row comes back with everything but the
+      // one thing that opens the frame, and the walk is where it gets it back.
+      // The row is put again — the bytes of the frame are still never read.
+      if (!seen.handle) {
+        batch.push({ ...seen, handle });
+        if (batch.length >= BATCH) await flush();
+      }
+      return;
+    }
    // A RAW is read whole because that is the only way LibRaw can seek to the
    // preview the camera wrote inside it; a JPEG is handed to the decoder as it
    // is, and only its header is read for the date. Both tiles come off what the
@@ -966,3 +976,83 @@ export async function listEditedIds(): Promise<Set<string>> {
    return new Set();
  }
 }
+
+export async function listEdits(): Promise<LibraryEdit[]> {
+  try {
+    return await ask<LibraryEdit[]>(EDITS, 'readonly', (s) => s.getAll());
+  } catch {
+    return [];
+  }
+}
+
+// --- restore ---------------------------------------------------------------
+
+// The catalogue as it travels: the rows without the two things that cannot be
+// written to a file — a handle, which only the visitor's picker can hand out, and
+// the tile itself, which is a blob and would be a second copy of the backup
+// folder. Everything else is here, `mtime` and `size` and `star` included, which
+// is what lets the walk that follows match every frame and read none of them.
+export type CatalogueRow = Omit<LibraryPhoto, 'handle' | 'thumb'>;
+
+// The rows a set of ids stands for, gaps left as gaps — one transaction for the
+// lot, where `getPhoto` would be one per frame.
+async function rowsAt(ids: string[]): Promise<(LibraryPhoto | undefined)[]> {
+  const db = await openDb();
+  return new Promise((resolve, reject) => {
+    const tx = db.transaction(PHOTOS, 'readonly');
+    const store = tx.objectStore(PHOTOS);
+    const out: (LibraryPhoto | undefined)[] = [];
+    ids.forEach((id, i) => {
+      const req = store.get(id);
+      req.onsuccess = () => {
+        out[i] = req.result as LibraryPhoto | undefined;
+      };
+    });
+    tx.oncomplete = () => resolve(out);
+    tx.onerror = () => reject(tx.error);
+  });
+}
+
+// Put a catalogue back. A frame the catalogue already holds and can open — one
+// with a handle — is left exactly as it is: the backup's copy of it would come
+// back without one, which is a working catalogue traded for a worse one. Every
+// other row goes in as it came out. `thumb` fetches the tile that was backed up
+// beside the row, and a tile that will not read leaves the row without one: the
+// grid paints a blank cell until the next scan reads the frame.
+export async function restoreCatalogue(
+  photos: CatalogueRow[],
+  edits: LibraryEdit[],
+  thumb?: (id: string) => Promise<Blob | null>
+): Promise<number> {
+  const db = await openDb();
+  const size = 200;
+  let put = 0;
+  for (let at = 0; at < photos.length; at += size) {
+    const slice = photos.slice(at, at + size);
+    const had = await rowsAt(slice.map((r) => r.id));
+    const want = slice.filter((_, i) => !had[i]?.handle);
+    if (!want.length) continue;
+    const rows = await Promise.all(
+      want.map(async (row) => ({ ...row, thumb: thumb ? await thumb(row.id) : null }) as unknown as LibraryPhoto)
+    );
+    await new Promise<void>((resolve, reject) => {
+      const tx = db.transaction(PHOTOS, 'readwrite');
+      const store = tx.objectStore(PHOTOS);
+      for (const row of rows) store.put(row);
+      tx.oncomplete = () => resolve();
+      tx.onerror = () => reject(tx.error);
+    });
+    put += rows.length;
+  }
+  for (let at = 0; at < edits.length; at += size) {
+    const slice = edits.slice(at, at + size);
+    await new Promise<void>((resolve, reject) => {
+      const tx = db.transaction(EDITS, 'readwrite');
+      const store = tx.objectStore(EDITS);
+      for (const row of slice) store.put(row);
+      tx.oncomplete = () => resolve();
+      tx.onerror = () => reject(tx.error);
+    });
+  }
+  return put;
+}
@@ -0,0 +1,318 @@
+// The catalogue's own copy, kept in a folder on the visitor's own disk. The
+// catalogue lives in the browser's storage, which is the one place a cleared
+// profile, a new machine or a reinstall takes away — and the frames themselves
+// are not in it, so what is lost is the reading: every tile, every star, every
+// recipe. This writes that reading out to a folder the visitor picks, and reads
+// it back into a catalogue that has nothing.
+//
+// The folder holds one file of rows — thumbnail, star, recipe, the frame's size
+// and write time, and no handle, which is not a thing that can be written down —
+// and the tiles beside it, one file per frame under `thumbs/<frame id>`, so the
+// folder mirrors the library exactly and a tile can be looked at, copied or
+// rsynced by hand. A frame that is being restored is matched to a frame on the
+// disk by that size and write time, so putting a catalogue back costs the tiles
+// and not one frame read.
+//
+// ponytail: no incremental tree walk, no deletes. A tile is written when the
+// frame is new or its tile changed, and one left behind by a frame that was
+// dropped from the library stays there — it is a few dozen kilobytes in a folder
+// the visitor owns. Add a sweep when a library that is edited down to a fraction
+// of its size makes the leftovers worth the walk.
+import type { LibraryEdit, LibraryFolder } from './library';
+import {
+  ensurePermission,
+  listEdits,
+  listFolders,
+  listPhotos,
+  renameFolder,
+  restoreCatalogue,
+  type CatalogueRow,
+} from './library';
+
+const DB_NAME = 'recipescam-backup';
+// Two stores of its own, not the library's: the library opens without a version
+// on purpose (see `openDb` there), and a version it has to reach for is one a
+// second tab can block. This one is nothing but a handle and a list, so it can
+// have a version and be upgraded.
+const WHERE = 'where';
+const WRITTEN = 'written';
+const DB_VERSION = 1;
+
+const CATALOGUE = 'recipescam-catalogue.json';
+const THUMBS = 'thumbs';
+// The catalogue is written whole however little changed: it is one file, and a
+// row that was left behind in it is a frame that will not come back.
+const VERSION = 1;
+
+interface Meta {
+  id: 'meta';
+  at: number;
+  photos: number;
+  wrote: number;
+}
+
+export interface BackupStatus {
+  // The folder's own name, or null when the visitor has not chosen one.
+  folder: string | null;
+  // Whether the browser has let the page write to it. It comes back `prompt`
+  // after a restart — a permission outlives the tab only while the tab does, and
+  // Chromium will not hand it back without a click — which is why the screen
+  // says so rather than failing silently.
+  granted: boolean;
+  at: number;
+  photos: number;
+  wrote: number;
+}
+
+export interface BackupProgress {
+  done: number;
+  total: number;
+}
+
+let dbPromise: Promise<IDBDatabase> | null = null;
+
+function openDb(): Promise<IDBDatabase> {
+  dbPromise ??= new Promise<IDBDatabase>((resolve, reject) => {
+    const req = indexedDB.open(DB_NAME, DB_VERSION);
+    req.onupgradeneeded = () => {
+      const db = req.result;
+      if (!db.objectStoreNames.contains(WHERE)) db.createObjectStore(WHERE, { keyPath: 'id' });
+      if (!db.objectStoreNames.contains(WRITTEN)) db.createObjectStore(WRITTEN, { keyPath: 'id' });
+    };
+    req.onsuccess = () => resolve(req.result);
+    req.onerror = () => reject(req.error);
+  });
+  return dbPromise;
+}
+
+function ask<T>(store: string, mode: IDBTransactionMode, run: (s: IDBObjectStore) => IDBRequest): Promise<T> {
+  return openDb().then(
+    (db) =>
+      new Promise<T>((resolve, reject) => {
+        const req = run(db.transaction(store, mode).objectStore(store));
+        req.onsuccess = () => resolve(req.result as T);
+        req.onerror = () => reject(req.error);
+      })
+  );
+}
+
+async function keep(store: string, rows: object[]): Promise<void> {
+  if (!rows.length) return;
+  const db = await openDb();
+  await new Promise<void>((resolve, reject) => {
+    const tx = db.transaction(store, 'readwrite');
+    const s = tx.objectStore(store);
+    for (const row of rows) s.put(row);
+    tx.oncomplete = () => resolve();
+    tx.onerror = () => reject(tx.error);
+  });
+}
+
+async function folderHandle(): Promise<FileSystemDirectoryHandle | null> {
+  try {
+    const row = await ask<{ id: string; handle: FileSystemDirectoryHandle } | undefined>(WHERE, 'readonly', (s) =>
+      s.get('folder')
+    );
+    return row?.handle ?? null;
+  } catch {
+    return null;
+  }
+}
+
+// The permission is asked for silently and never granted from here: a prompt
+// without a click behind it is a prompt the browser may refuse outright, and the
+// buttons on the screen are where the click is.
+async function granted(handle: FileSystemHandle, write: boolean): Promise<boolean> {
+  const askable = handle as FileSystemHandle & {
+    queryPermission?: (o: { mode: string }) => Promise<PermissionState>;
+  };
+  try {
+    return (await askable.queryPermission?.({ mode: write ? 'readwrite' : 'read' })) === 'granted';
+  } catch {
+    return false;
+  }
+}
+
+export async function backupStatus(): Promise<BackupStatus> {
+  const handle = await folderHandle();
+  const meta = await ask<Meta | undefined>(WHERE, 'readonly', (s) => s.get('meta')).catch(() => undefined);
+  return {
+    folder: handle?.name ?? null,
+    granted: handle ? await granted(handle, true) : false,
+    at: meta?.at ?? 0,
+    photos: meta?.photos ?? 0,
+    wrote: meta?.wrote ?? 0,
+  };
+}
+
+// The picker's own dialog, the same way a folder of frames is picked — a second
+// `id` so the browser remembers this folder apart from the library's.
+export async function pickBackupFolder(): Promise<string> {
+  if (!('showDirectoryPicker' in window)) throw new Error('no-folder-picker');
+  const handle = await (
+    window as unknown as {
+      showDirectoryPicker: (o?: unknown) => Promise<FileSystemDirectoryHandle>;
+    }
+  ).showDirectoryPicker({ mode: 'readwrite', id: 'recipescam-backup' });
+  const before = await folderHandle();
+  await keep(WHERE, [{ id: 'folder', handle }]);
+  // A folder that is not the one the last run wrote into starts empty, and the
+  // list of tiles already written is about that folder — kept, it would say every
+  // tile is there and the new folder would come out with none of them.
+  if (before?.name !== handle.name) await ask(WRITTEN, 'readwrite', (s) => s.clear());
+  return handle.name;
+}
+
+// A handle comes back from IndexedDB without its permission, so every write goes
+// through this: the visitor's click is the one thing that can hand it back.
+async function writable(): Promise<FileSystemDirectoryHandle> {
+  const handle = await folderHandle();
+  if (!handle) throw new Error('no-backup-folder');
+  if (!(await ensurePermission(handle, true))) throw new Error('no-backup-permission');
+  return handle;
+}
+
+// The directories are asked for by the path in the frame id, and the id of every
+// frame in a folder starts with that folder's: a cache of what has been opened
+// turns a folder of 220 000 frames into a walk of the folders it is made of.
+const opened = new WeakMap<FileSystemDirectoryHandle, Map<string, FileSystemDirectoryHandle>>();
+
+async function at(root: FileSystemDirectoryHandle, rel: string, create = false): Promise<FileSystemDirectoryHandle> {
+  let cache = opened.get(root);
+  if (!cache) opened.set(root, (cache = new Map()));
+  let node = root;
+  let path = '';
+  for (const part of rel.split('/')) {
+    if (!part) continue;
+    path = path ? `${path}/${part}` : part;
+    const had = cache.get(path);
+    if (had) {
+      node = had;
+      continue;
+    }
+    node = await node.getDirectoryHandle(part, { create });
+    cache.set(path, node);
+  }
+  return node;
+}
+
+async function put(root: FileSystemDirectoryHandle, rel: string, blob: Blob): Promise<void> {
+  const cut = rel.lastIndexOf('/');
+  const dir = cut < 0 ? root : await at(root, rel.slice(0, cut), true);
+  const file = await dir.getFileHandle(cut < 0 ? rel : rel.slice(cut + 1), { create: true });
+  // One write at a time and closed at once: a writable that is left open is a
+  // file that is not on the disk yet, and a backup that is interrupted is one the
+  // next run has to write again.
+  const writer = await file.createWritable();
+  await writer.write(blob);
+  await writer.close();
+}
+
+// What the last run left in the folder: the tile's own size, per frame. A tile
+// that has not changed is not written again, which is the whole of what makes a
+// second run cheap.
+async function already(): Promise<Map<string, number>> {
+  const rows = await ask<{ id: string; size: number }[]>(WRITTEN, 'readonly', (s) => s.getAll()).catch(() => []);
+  return new Map(rows.map((r) => [r.id, r.size]));
+}
+
+export interface BackupResult extends BackupStatus {
+  // The tiles this run actually wrote, and the frames it found.
+  written: number;
+}
+
+export async function backupNow(onProgress?: (p: BackupProgress) => void): Promise<BackupResult> {
+  const root = await writable();
+  const [photos, edits, folders] = await Promise.all([listPhotos(), listEdits(), listFolders()]);
+  const had = await already();
+  const at0 = Date.now();
+  const cat = {
+    version: VERSION,
+    at: at0,
+    folders: folders.map(({ name, label }) => ({ name, label })),
+    // The two things a file cannot hold: the handle, and the tile, which is
+    // written beside this as a file of its own.
+    photos: photos.map(({ handle, thumb, ...row }) => row) as CatalogueRow[],
+    edits,
+  };
+  await put(root, CATALOGUE, new Blob([JSON.stringify(cat)], { type: 'application/json' }));
+
+  let written = 0;
+  let done = 0;
+  const rows: { id: string; size: number }[] = [];
+  for (const photo of photos) {
+    // Progress counts frames and not tiles: a folder of frames read without one
+    // is still a reading, and a bar that does not move is a bar that reads as a
+    // screen that has stopped.
+    if (++done % 64 === 0) onProgress?.({ done, total: photos.length });
+    if (!photo.thumb) continue;
+    if (had.get(photo.id) === photo.thumb.size) continue;
+    const parts = photo.id.split('/');
+    parts.pop();
+    const dir = await at(root, [THUMBS, ...parts].join('/'), true);
+    const file = await dir.getFileHandle(photo.name, { create: true });
+    const writer = await file.createWritable();
+    await writer.write(photo.thumb);
+    await writer.close();
+    rows.push({ id: photo.id, size: photo.thumb.size });
+    written++;
+    // The list is kept in step with the writes: a run that is interrupted
+    // continues from the frame it stopped at instead of writing the folder
+    // again.
+    if (rows.length >= 200) await keep(WRITTEN, rows.splice(0, rows.length));
+  }
+  await keep(WRITTEN, rows);
+  onProgress?.({ done: photos.length, total: photos.length });
+  const meta: Meta = { id: 'meta', at: at0, photos: photos.length, wrote: written };
+  await keep(WHERE, [meta]);
+  return { folder: root.name, granted: true, at: at0, photos: photos.length, wrote: written, written };
+}
+
+export interface RestoreResult {
+  photos: number;
+  at: number;
+}
+
+// Put a catalogue back. Nothing is deleted and nothing that is already there is
+// spared: the backup is the reading, and a frame it knows about is a frame the
+// catalogue should know about too. Folders are not in it — a folder is a handle,
+// and a handle only comes from the visitor's own picker — so the frames come back
+// without one and the folders are picked again afterwards, which is also what
+// gives every frame its handle back without reading a byte of it.
+export async function restoreNow(onProgress?: (p: BackupProgress) => void): Promise<RestoreResult> {
+  const root = await folderHandle();
+  if (!root) throw new Error('no-backup-folder');
+  if (!(await ensurePermission(root, false))) throw new Error('no-backup-permission');
+  const text = await (await (await root.getFileHandle(CATALOGUE)).getFile()).text();
+  const cat = JSON.parse(text) as {
+    at?: number;
+    folders?: { name: string; label?: string }[];
+    photos: CatalogueRow[];
+    edits: LibraryEdit[];
+  };
+  // The folders this catalogue still has keep the names the backup remembers:
+  // a rename is the reader's own, and it is not in the frame's file anywhere.
+  const here = new Map<string, LibraryFolder>((await listFolders()).map((f) => [f.name, f]));
+  for (const f of cat.folders ?? []) {
+    const folder = here.get(f.name);
+    if (folder && f.label && folder.label !== f.label) await renameFolder(folder, f.label);
+  }
+  let done = 0;
+  const photos = await restoreCatalogue(cat.photos ?? [], cat.edits ?? [], async (id) => {
+    if (++done % 64 === 0) onProgress?.({ done, total: (cat.photos ?? []).length });
+    try {
+      const parts = id.split('/');
+      const name = parts.pop() as string;
+      const dir = await at(root, [THUMBS, ...parts].join('/'));
+      const file = await dir.getFileHandle(name);
+      return new Blob([await (await file.getFile()).arrayBuffer()], { type: 'image/jpeg' });
+    } catch {
+      // A tile that is not there — or that has been taken away by hand — leaves
+      // the row without one, and the next scan of the folder reads the frame and
+      // paints it. Losing the picture is not worth losing the reading.
+      return null;
+    }
+  });
+  onProgress?.({ done: photos, total: photos });
+  return { photos, at: cat.at ?? 0 };
+}