RecipesCam/docker/frontend/src/engine/libraryBackup.ts

// The catalogue's own copy, kept in a folder on the visitor's own disk. The
// catalogue lives in the browser's storage, which is the one place a cleared
// profile, a new machine or a reinstall takes away — and the frames themselves
// are not in it, so what is lost is the reading: every tile, every star, every
// recipe. This writes that reading out to a folder the visitor picks, and reads
// it back into a catalogue that has nothing.
//
// The folder holds one file of rows — thumbnail, star, recipe, the frame's size
// and write time, and no handle, which is not a thing that can be written down —
// and the tiles beside it, one file per frame under `thumbs/<frame id>` — a
// component the browser refuses to write spelled out as `%XX` (see `spell`) so
// the tiles of such a folder land too — so the
// folder mirrors the library exactly and a tile can be looked at, copied or
// rsynced by hand. A frame that is being restored is matched to a frame on the
// disk by that size and write time, so putting a catalogue back costs the tiles
// and not one frame read.
//
// ponytail: no incremental tree walk, no deletes. A tile is written when the
// frame is new or its tile changed, and one left behind by a frame that was
// dropped from the library stays there — it is a few dozen kilobytes in a folder
// the visitor owns. Add a sweep when a library that is edited down to a fraction
// of its size makes the leftovers worth the walk.
import type { LibraryEdit, LibraryFolder } from './library';
import {
  ensurePermission,
  forgetWalkedDirs,
  getPhotoThumb,
  listEdits,
  listFolders,
  listPhotos,
  renameFolder,
  restoreCatalogue,
  setBackupTileFetcher,
  type CatalogueRow,
} from './library';

const DB_NAME = 'recipescam-backup';
// Two stores of its own, not the library's: the library opens without a version
// on purpose (see `openDb` there), and a version it has to reach for is one a
// second tab can block. This one is nothing but a handle and a list, so it can
// have a version and be upgraded.
const WHERE = 'where';
const WRITTEN = 'written';
const DB_VERSION = 1;

const CATALOGUE = 'recipescam-catalogue.json';
const THUMBS = 'thumbs';
// The catalogue is written whole however little changed: it is one file, and a
// row that was left behind in it is a frame that will not come back.
const VERSION = 1;

export async function readBackupTile(id: string): Promise<Blob | null> {
  try {
    const root = await folderHandle();
    if (!root) return null;
    const parts = id.split('/');
    const name = parts.pop() as string;
    const dir = await at(root, [THUMBS, ...parts].join('/'));
    const file = await dir.getFileHandle(spell(name));
    return new Blob([await (await file.getFile()).arrayBuffer()], { type: 'image/jpeg' });
  } catch {
    return null;
  }
}

// Automatically register as lazy backup tile fetcher for library.ts
setBackupTileFetcher(readBackupTile);

interface Meta {
  id: 'meta';
  at: number;
  photos: number;
  wrote: number;
}

export interface BackupStatus {
  // The folder's own name, or null when the visitor has not chosen one.
  folder: string | null;
  // Whether the browser has let the page write to it. It comes back `prompt`
  // after a restart — a permission outlives the tab only while the tab does, and
  // Chromium will not hand it back without a click — which is why the screen
  // says so rather than failing silently.
  granted: boolean;
  at: number;
  photos: number;
  wrote: number;
}

export interface BackupProgress {
  done: number;
  total: number;
}

let dbPromise: Promise<IDBDatabase> | null = null;

function openDb(): Promise<IDBDatabase> {
  dbPromise ??= new Promise<IDBDatabase>((resolve, reject) => {
    const req = indexedDB.open(DB_NAME, DB_VERSION);
    req.onupgradeneeded = () => {
      const db = req.result;
      if (!db.objectStoreNames.contains(WHERE)) db.createObjectStore(WHERE, { keyPath: 'id' });
      if (!db.objectStoreNames.contains(WRITTEN)) db.createObjectStore(WRITTEN, { keyPath: 'id' });
    };
    req.onsuccess = () => resolve(req.result);
    req.onerror = () => reject(req.error);
  });
  return dbPromise;
}

function ask<T>(store: string, mode: IDBTransactionMode, run: (s: IDBObjectStore) => IDBRequest): Promise<T> {
  return openDb().then(
    (db) =>
      new Promise<T>((resolve, reject) => {
        const req = run(db.transaction(store, mode).objectStore(store));
        req.onsuccess = () => resolve(req.result as T);
        req.onerror = () => reject(req.error);
      })
  );
}

async function keep(store: string, rows: object[]): Promise<void> {
  if (!rows.length) return;
  const db = await openDb();
  await new Promise<void>((resolve, reject) => {
    const tx = db.transaction(store, 'readwrite');
    const s = tx.objectStore(store);
    for (const row of rows) s.put(row);
    tx.oncomplete = () => resolve();
    tx.onerror = () => reject(tx.error);
  });
}

async function folderHandle(): Promise<FileSystemDirectoryHandle | null> {
  try {
    const row = await ask<{ id: string; handle: FileSystemDirectoryHandle } | undefined>(WHERE, 'readonly', (s) =>
      s.get('folder')
    );
    return row?.handle ?? null;
  } catch {
    return null;
  }
}

// The permission is asked for silently and never granted from here: a prompt
// without a click behind it is a prompt the browser may refuse outright, and the
// buttons on the screen are where the click is.
async function granted(handle: FileSystemHandle, write: boolean): Promise<boolean> {
  if (!handle) return false;
  const askable = handle as FileSystemHandle & {
    queryPermission?: (o: { mode: string }) => Promise<PermissionState>;
  };
  try {
    return (await askable.queryPermission?.({ mode: write ? 'readwrite' : 'read' })) === 'granted';
  } catch {
    return false;
  }
}

export async function backupStatus(): Promise<BackupStatus> {
  const handle = await folderHandle();
  const meta = await ask<Meta | undefined>(WHERE, 'readonly', (s) => s.get('meta')).catch(() => undefined);
  return {
    folder: handle?.name ?? null,
    granted: handle ? await granted(handle, true) : false,
    at: meta?.at ?? 0,
    photos: meta?.photos ?? 0,
    wrote: meta?.wrote ?? 0,
  };
}

// The catalogue's own header, read out of the file and not out of this browser's
// memory of it. A folder the reader is pointing at may be a copy carried over on
// a stick, written by another machine, at another time: it is its own catalogue
// that says which backup it is, and its own count and date that the screen has
// to show before anything is written to it or read from it.
async function head(root: FileSystemDirectoryHandle): Promise<Meta | null> {
  try {
    const text = await (await (await root.getFileHandle(CATALOGUE)).getFile()).text();
    const cat = JSON.parse(text) as { at?: number; photos?: unknown[] };
    return { id: 'meta', at: cat.at ?? 0, photos: (cat.photos ?? []).length, wrote: 0 };
  } catch {
    return null;
  }
}

// The picker's own dialog, the same way a folder of frames is picked — a second
// `id` so the browser remembers this folder apart from the library's. It is the
// only way a folder is ever chosen, and the only thing that hands a remembered
// one its permission back: the browser takes that away when the tab closes, and
// a permission asked for without a picker behind it is one it may refuse.
export async function pickBackupFolder(): Promise<string> {
  if (!('showDirectoryPicker' in window)) throw new Error('no-folder-picker');
  const handle = await (
    window as unknown as {
      showDirectoryPicker: (o?: unknown) => Promise<FileSystemDirectoryHandle>;
    }
  ).showDirectoryPicker({ mode: 'readwrite', id: 'recipescam-backup' });
  const before = await folderHandle();
  await keep(WHERE, [{ id: 'folder', handle }]);
  // A folder that is not the one the last run wrote into starts empty, and the
  // list of tiles already written is about that folder — kept, it would say every
  // tile is there and the new folder would come out with none of them.
  if (before?.name !== handle.name) {
    await ask(WRITTEN, 'readwrite', (s) => s.clear());
    // And the row goes back to saying what this folder holds rather than what
    // the last one did: a count and a date from the folder next door is a count
    // and a date about a backup the reader is not looking at.
    const meta = await head(handle);
    await ask<Meta | undefined>(WHERE, 'readwrite', (s) => (meta ? s.put(meta) : s.delete('meta')));
  }
  return handle.name;
}

// A handle comes back from IndexedDB without its permission, so every write goes
// through this: the visitor's click is the one thing that can hand it back.
async function writable(): Promise<FileSystemDirectoryHandle> {
  const handle = await folderHandle();
  if (!handle) throw new Error('no-backup-folder');
  if (!(await ensurePermission(handle, true))) throw new Error('no-backup-permission');
  return handle;
}

// Chrome will not spell a name it will not put on a disk: no `* ? " < > | : \ /`,
// no control or format character, no space, dot or tilde at either end, no
// `.lnk` `.scf` `.url`, and it has to be UTF-8 — the whole of what Chromium's
// `IsSafePathComponent` refuses, and every one of those is a name a camera or a
// person is free to give a folder. The library is read through a handle, which
// lets every one of them through, so the refusal only ever surfaces here, on the
// way out: `getDirectoryHandle` says `Name is not allowed` and the whole backup
// stops. This spells such a name instead — the offending character as `%XX` of
// its UTF-8 bytes — and it is the same function on the way back in, so a tile
// under a folder called `2024 : ảnh.` still comes home. A name that was already
// safe is handed back untouched, which is what keeps the folder a person can
// read, copy or rsync by hand.
const UNSAFE = /[<>:"|?*\\/]|\p{Cc}|\p{Cf}|[\uFDD0-\uFDEF\uFFFE\uFFFF]/u;
const EDGE = /^[ .~]|[ .~]$/;
const SHORTCUT = /\.(lnk|scf|url)$/i;

const hex = (ch: string) =>
  [...new TextEncoder().encode(ch)].map((b) => `%${b.toString(16).toUpperCase().padStart(2, '0')}`).join('');

function spell(name: string): string {
  // `%` spells here even though a disk takes it: a name that holds one would
  // otherwise spell the same as a name whose character was spelled out, and the
  // two tiles would land on each other's file.
  if (
    name &&
    name !== '.' &&
    name !== '..' &&
    !name.includes('%') &&
    !UNSAFE.test(name) &&
    !EDGE.test(name) &&
    !SHORTCUT.test(name)
  ) {
    return name;
  }
  const chars = [...name];
  return chars
    .map((ch, i) => {
      const edge = (i === 0 || i === chars.length - 1) && /[ .~]/.test(ch);
      return ch === '%' || edge || UNSAFE.test(ch) ? hex(ch) : ch;
    })
    .join('')
    .replace(SHORTCUT, (m) => `%2E${m.slice(1)}`);
}

// The directories are asked for by the path in the frame id, and the id of every
// frame in a folder starts with that folder's: a cache of what has been opened
// turns a folder of 220 000 frames into a walk of the folders it is made of.
const opened = new WeakMap<FileSystemDirectoryHandle, Map<string, FileSystemDirectoryHandle>>();

async function at(root: FileSystemDirectoryHandle, rel: string, create = false): Promise<FileSystemDirectoryHandle> {
  let cache = opened.get(root);
  if (!cache) opened.set(root, (cache = new Map()));
  let node = root;
  let path = '';
  for (const part of rel.split('/')) {
    if (!part) continue;
    const name = spell(part);
    path = path ? `${path}/${name}` : name;
    const had = cache.get(path);
    if (had) {
      node = had;
      continue;
    }
    node = await node.getDirectoryHandle(name, { create });
    cache.set(path, node);
  }
  return node;
}

async function put(root: FileSystemDirectoryHandle, rel: string, blob: Blob): Promise<void> {
  const cut = rel.lastIndexOf('/');
  const dir = cut < 0 ? root : await at(root, rel.slice(0, cut), true);
  const file = await dir.getFileHandle(spell(cut < 0 ? rel : rel.slice(cut + 1)), { create: true });
  // One write at a time and closed at once: a writable that is left open is a
  // file that is not on the disk yet, and a backup that is interrupted is one the
  // next run has to write again.
  const writer = await file.createWritable();
  await writer.write(blob);
  await writer.close();
}

// What the last run left in the folder: the tile's own size, per frame. A tile
// that has not changed is not written again, which is the whole of what makes a
// second run cheap.
async function already(): Promise<Map<string, number>> {
  const rows = await ask<{ id: string; size: number }[]>(WRITTEN, 'readonly', (s) => s.getAll()).catch(() => []);
  return new Map(rows.map((r) => [r.id, r.size]));
}

export interface BackupResult extends BackupStatus {
  // The tiles this run actually wrote, and the frames it found.
  written: number;
}

export async function backupNow(onProgress?: (p: BackupProgress) => void): Promise<BackupResult> {
  const root = await writable();
  const [all, edits, allFolders] = await Promise.all([listPhotos(), listEdits(), listFolders()]);
  // The copy is the disk's: a node that is really an Immich server and the frames
  // read over the network from it are not on this machine, so they are not part of
  // what a folder on this machine can hold. The catalogue still keeps them — see
  // `immich.ts`, which reads them again on the next visit.
  const photos = all.filter((p) => !p.remoteId);
  const folders = allFolders.filter((f) => !f.source);
  const had = await already();
  const at0 = Date.now();
  const cat = {
    version: VERSION,
    at: at0,
    folders: folders.map(({ name, label }) => ({ name, label })),
    // The two things a file cannot hold: the handle, and the tile, which is
    // written beside this as a file of its own.
    photos: photos.map(({ handle, thumb, ...row }) => row) as CatalogueRow[],
    edits,
  };
  await put(root, CATALOGUE, new Blob([JSON.stringify(cat)], { type: 'application/json' }));

  let written = 0;
  let skipped = 0;
  let done = 0;
  const rows: { id: string; size: number }[] = [];
  for (const photo of photos) {
    if (++done % 64 === 0) onProgress?.({ done, total: photos.length });
    const blob = photo.thumb ?? (await getPhotoThumb(photo.id));
    if (!blob) continue;
    if (had.get(photo.id) === blob.size) continue;
    const parts = photo.id.split('/');
    parts.pop();
    const rel = [THUMBS, ...parts, photo.name].join('/');
    try {
      await put(root, rel, blob);
    } catch (err) {
      if (skipped < 5) console.warn('backup: could not write', rel, err);
      skipped++;
      continue;
    }
    rows.push({ id: photo.id, size: blob.size });
    written++;
    if (rows.length >= 200) await keep(WRITTEN, rows.splice(0, rows.length));
  }
  await keep(WRITTEN, rows);
  if (skipped) console.warn(`backup: ${skipped} of ${photos.length} tiles skipped`);
  onProgress?.({ done: photos.length, total: photos.length });
  const meta: Meta = { id: 'meta', at: at0, photos: photos.length, wrote: written };
  await keep(WHERE, [meta]);
  return { folder: root.name, granted: true, at: at0, photos: photos.length, wrote: written, written };
}

export interface RestoreResult {
  photos: number;
  at: number;
}

// Put a catalogue back. Nothing is deleted and nothing that is already there is
// spared: the backup is the reading, and a frame it knows about is a frame the
// catalogue should know about too. Folders are not in it — a folder is a handle,
// and a handle only comes from the visitor's own picker — so the frames come back
// without one and the folders are picked again afterwards, which is also what
// gives every frame its handle back without reading a byte of it.
export async function restoreNow(onProgress?: (p: BackupProgress) => void): Promise<RestoreResult> {
  const root = await folderHandle();
  if (!root) throw new Error('no-backup-folder');
  if (!(await ensurePermission(root, false))) throw new Error('no-backup-permission');
  const text = await (await (await root.getFileHandle(CATALOGUE)).getFile()).text();
  const cat = JSON.parse(text) as {
    at?: number;
    folders?: { name: string; label?: string }[];
    photos: CatalogueRow[];
    edits: LibraryEdit[];
  };
  // The folders this catalogue still has keep the names the backup remembers:
  // a rename is the reader's own, and it is not in the frame's file anywhere.
  const here = new Map<string, LibraryFolder>((await listFolders()).map((f) => [f.name, f]));
  for (const f of cat.folders ?? []) {
    const folder = here.get(f.name);
    if (folder && f.label && folder.label !== f.label) await renameFolder(folder, f.label);
  }

  const photoList = cat.photos ?? [];
  const editList = cat.edits ?? [];
  const total = photoList.length;

  // Instant catalog metadata & edits restoration (< 100ms)
  const photos = await restoreCatalogue(photoList, editList);
  // And the walks are forgotten with it: the numbers a walk wrote down are off a
  // disk this backup does not describe, and a column drawn under them would say a
  // folder holds what it held the last time the reader looked at it.
  await forgetWalkedDirs();

  // Activate backup tile fetcher for lazy rendering on screen
  setBackupTileFetcher(readBackupTile);

  onProgress?.({ done: total, total });
  return { photos, at: cat.at ?? 0 };
}