fix(library): land a reading's rows as they are read, and keep the copy the reader reads offline

- scan: rows go down on a clock as well as on a full batch. The reader's folder
  of 26 RAWs is under one batch of 50, so nothing of it reached the screen until
  the reading was over — a wall that stood empty for the whole scan and then
  filled in one go. Frames now reach the screen every 400ms while the roll is
  still being read, so the count climbs and the first tiles are drawn as the
  frames land.
- tiles: a HEIC or a RAW has its tile written the first time it is drawn, so
  every later visit reads the tile back instead of the file.
- resume: a reading a reload cut off is offered by the next visit with a chip
  instead of being run behind the reader's back.
- offline listing: the desk's listing is kept whole in IndexedDB under
  `recipescam-offline`/`reads`, past the 200 000 characters localStorage held,
  so an offline visit opens on the listing it kept instead of on nothing.
- offline writes: a vote or a view cast with no server is held and goes out when
  one answers again, with a bar that says how many are waiting.
- sw: the shell precaches its own bundle, stylesheet and icon, read out of the
  document — a worker that precached only the document answered an offline
  launch with a page whose JavaScript the machine did not have.
- checks in docker/frontend/scripts: tree-scan-check (counts while reading,
  resume after a reload, tiles of a long roll), offline-read-check,
  offline-queue-check.

docker/frontend/src/engine/library.ts

@@ -23,7 +23,8 @@ const FOLDERS = 'folders';
 const PHOTOS = 'photos';
 const THUMBS = 'thumbs';
 const EDITS = 'edits';
-const DB_VERSION = 2;
+const DIRS = 'dirs';
+const DB_VERSION = 3;
 
 // The tile the catalogue keeps is the same tile the grid cell and the strip both
 // paint, and there is one of them per frame of a roll that reaches six figures.
@@ -38,6 +39,15 @@ const THUMB_QUALITY = 0.8;
 // Frames go down in batches, so a 3000-file folder is 60 transactions rather
 // than 3000 of them.
 const BATCH = 50;
+// A batch that only goes down when it is full is a wall that stands empty for the
+// whole of a reading and then fills in one go: the reader's folder of 26 RAWs is
+// under one batch, so nothing of it reached the screen until the reading was over.
+// The clock is the other end of the batch — whatever is in hand goes down once this
+// long has passed since the last time, so the count climbs and the first tiles are
+// drawn while the roll is still being read.
+// ponytail: 400ms of the reading's own clock, no timer to start or stop; a roll of
+// twenty thousand frames is fifty extra transactions on a reading of minutes.
+const FLUSH_MS = 400;
 // Frames read at once. Reading a frame is nearly all waiting — the bytes come off
 // the disk, the decode runs on a thread of its own — and a wait that is not spent
 // on the next frame is time the roll does not get back. Past a few, the disk and
@@ -236,6 +246,7 @@ function openDb(): Promise<IDBDatabase> {
         db.createObjectStore(THUMBS, { keyPath: 'id' });
       }
       if (!db.objectStoreNames.contains(EDITS)) db.createObjectStore(EDITS, { keyPath: 'photoId' });
+      if (!db.objectStoreNames.contains(DIRS)) db.createObjectStore(DIRS, { keyPath: 'folder' });
     };
     req.onsuccess = () => {
       blockedSink?.(false);
@@ -413,9 +424,11 @@ export async function removeFolder(name: string): Promise<void> {
   const db = await openDb();
   await new Promise<void>((resolve, reject) => {
     const hasThumbs = db.objectStoreNames.contains(THUMBS);
-    const stores = hasThumbs ? [FOLDERS, PHOTOS, THUMBS, EDITS] : [FOLDERS, PHOTOS, EDITS];
+    const hasDirs = db.objectStoreNames.contains(DIRS);
+    const stores = [FOLDERS, PHOTOS, EDITS, ...(hasThumbs ? [THUMBS] : []), ...(hasDirs ? [DIRS] : [])];
     const tx = db.transaction(stores, 'readwrite');
     tx.objectStore(FOLDERS).delete(name);
+    if (hasDirs) tx.objectStore(DIRS).delete(name);
     const photos = tx.objectStore(PHOTOS);
     const thumbs = hasThumbs ? tx.objectStore(THUMBS) : null;
     const edits = tx.objectStore(EDITS);
@@ -433,6 +446,192 @@ export async function removeFolder(name: string): Promise<void> {
   await clearWalk(name);
 }
 
+// The frames a reading met are the frames the disk holds. The catalogue keeps a
+// row for every frame it has ever seen and nothing ever took one out: a frame
+// taken off the disk kept its row, and a folder with no reading of its own falls
+// back on the catalogue for a number — a number that could only climb. And the
+// row is not only a number: it is a tile the wall draws and a frame the strip
+// offers, one whose file is not there. A reading that listed the whole of what
+// it was pointed at is the one thing that knows the disk, so what it did not
+// meet is dropped here — the row, its tile, and the marks on it, because a star
+// or a rotation belongs to a frame that is there.
+//
+// `from` is the folder the reading was pointed at, '' for the whole roll; `met`
+// is every frame it found, spelled relative to the roll.
+async function prunePhotos(folder: string, from: string, met: Set<string>): Promise<void> {
+  try {
+    const root = normPath(folder);
+    const under = (from ? `${root}/${normPath(from)}` : root).toLowerCase();
+    const rows = (await listPhotos(folder)).filter((p) => {
+      const rel = (p.dir ? `${normPath(p.dir)}/${p.name}` : p.name).toLowerCase();
+      const key = `${root.toLowerCase()}/${rel}`;
+      if (from && key !== under && !key.startsWith(`${under}/`)) return false;
+      return !met.has(rel);
+    });
+    if (!rows.length) return;
+    const gone = new Set(rows.map((p) => p.id));
+    if (cached) cached = cached.filter((p) => !gone.has(p.id));
+    const db = await openDb();
+    const hasThumbs = db.objectStoreNames.contains(THUMBS);
+    await new Promise<void>((resolve, reject) => {
+      const tx = db.transaction([PHOTOS, EDITS, ...(hasThumbs ? [THUMBS] : [])], 'readwrite');
+      const photos = tx.objectStore(PHOTOS);
+      const edits = tx.objectStore(EDITS);
+      const thumbs = hasThumbs ? tx.objectStore(THUMBS) : null;
+      for (const p of rows) {
+        photos.delete(p.id);
+        edits.delete(p.id);
+        thumbs?.delete(p.id);
+        thumbCache.delete(p.id);
+      }
+      tx.oncomplete = () => resolve();
+      tx.onerror = () => reject(tx.error);
+    });
+  } catch {
+    // A repair that failed leaves the reading's own word standing: the counts
+    // are written either way, and the rows it did not meet are the next
+    // reading's to drop.
+  }
+}
+
+// --- what the walk found ----------------------------------------------------
+
+// The folders a reading walked into, and how many frames it met under each, kept
+// where the catalogue is kept. The column is drawn from this list, and a list
+// that lived only for the length of the reading took the rows with it: the reader
+// came back to a roll missing every folder that held no frame the catalogue had —
+// a folder of frames the reading never reached, and an empty folder alike — and
+// had to walk their parent a second time to get the names back. The numbers come
+// from the walk for the same reason: a folder the reading never reached is a
+// folder whose size the catalogue cannot say, and the walk is the only thing that
+// can.
+//
+// `counts` is the reading's own map, folded to one spelling per path, which is
+// how the screen looks a number up.
+export interface WalkedDirs {
+  // The picked folder, by name.
+  folder: string;
+  // Every folder the walk has been into, `rel` spelled the way the walk spells
+  // it: `''` for the roll itself is not a row, so it is not one here.
+  dirs: LibraryDir[];
+  // Folder path in the reading's own spelling, lower-cased on the way in, to the
+  // number of frames the walk met under it.
+  counts: Record<string, number>;
+}
+
+export async function listWalkedDirs(): Promise<WalkedDirs[]> {
+  try {
+    const rows = await ask<WalkedDirs[]>(DIRS, 'readonly', (s) => s.getAll());
+    return rows.map((r) => ({ folder: r.folder, dirs: r.dirs ?? [], counts: r.counts ?? {} }));
+  } catch {
+    return [];
+  }
+}
+
+// What the walk found, written down where the screen can pick it up. A reading
+// kept to one folder knows the folders under that one and nothing of the rest of
+// the roll, so it is folded into what is already there; a reading of the whole
+// roll is the roll, and what it did not find is not there any more.
+export async function saveWalkedDirs(
+  folder: string,
+  dirs: LibraryDir[],
+  counts: Record<string, number> | null,
+  replace: boolean,
+  from = '',
+  // The folders the reading asked to open and could not, spelled the way the walk
+  // spells them. A reading has no word on these, and a nought beside the name of
+  // one of them is a word it does not have.
+  unlisted?: Iterable<string>
+): Promise<void> {
+  try {
+    const root = normPath(folder).toLowerCase();
+    // One spelling to a path: the reading's own map holds every key twice, as the
+    // walk spelled it and in lower case, and a branch counted off both is twice
+    // its own size. Folded here once, and the record keeps the one spelling the
+    // screen looks a number up by.
+    const fold = (m: Record<string, number>): Record<string, number> => {
+      const out: Record<string, number> = {};
+      for (const [k, n] of Object.entries(m)) out[k.replace(/\\/g, '/').toLowerCase()] = n;
+      return out;
+    };
+    // The folders this reading was in, before another visit's are folded in.
+    const walked = dirs.slice();
+    if (!replace) {
+      const prev = (await ask<WalkedDirs | undefined>(DIRS, 'readonly', (s) => s.get(folder))) ?? null;
+      const byId = new Map((prev?.dirs ?? []).map((d) => [d.id, d]));
+      for (const d of dirs) byId.set(d.id, d);
+      dirs = [...byId.values()];
+      if (!counts) {
+        counts = fold(prev?.counts ?? {});
+      } else {
+        const was = fold(prev?.counts ?? {});
+        const now = fold(counts);
+        // A reading kept to one folder counts the frames under it and says
+        // nothing of the rest of the roll — but the rows above it hold those
+        // frames too. Written in as they came, the branch would move and its
+        // parents would not, a folder with more under it than the folder it sits
+        // in and a roll under its own tree. So the difference the branch came by
+        // is the difference each row above it comes by, and the reading's numbers
+        // take the place of the ones the last reading left under it — a branch
+        // the reader emptied is a branch this reading met nothing in.
+        const sub = from ? `${root}/${normPath(from).toLowerCase()}` : root;
+        const under = (k: string) => k === sub || k.startsWith(`${sub}/`);
+        const sum = (m: Record<string, number>) => {
+          let n = 0;
+          for (const [k, v] of Object.entries(m)) if (under(k)) n += v;
+          return n;
+        };
+        const delta = sum(now) - sum(was);
+        const merged: Record<string, number> = {};
+        for (const [k, v] of Object.entries(was)) if (!under(k)) merged[k] = v;
+        Object.assign(merged, now);
+        for (let anc = sub; delta && anc.length > root.length; ) {
+          anc = anc.slice(0, anc.lastIndexOf('/'));
+          merged[anc] = (merged[anc] ?? 0) + delta;
+        }
+        counts = merged;
+      }
+    }
+    if (counts) {
+      // A folder the reading was in and met no frame under is a folder with
+      // nothing under it: left out of the record, the row falls back on the
+      // catalogue, and a frame taken off the disk keeps its number for good.
+      const key = (rel: string) => (rel ? `${root}/${normPath(rel)}` : root).toLowerCase();
+      // A folder the reading could not open is a folder it has no word on, and
+      // nothing — not even nothing — is what a reading with no word on a folder
+      // may write beside its name. Written in as a nought it takes the place of
+      // the number the catalogue holds, and a column drawn off a drive that
+      // hiccupped once reads noughts over folders full of frames until the reader
+      // rescans the roll. Left out, the row falls back on the catalogue, which is
+      // what the last reading that could open it found.
+      const unsaid = new Set([...(unlisted ?? [])].map((r) => normPath(r).toLowerCase()));
+      for (const d of walked) {
+        if (unsaid.has(normPath(d.rel).toLowerCase())) continue;
+        counts[key(d.rel)] ??= 0;
+      }
+      if (replace) counts[root] ??= 0;
+      counts = fold(counts);
+    }
+    await ask(DIRS, 'readwrite', (s) => s.put({ folder, dirs, counts: counts ?? {} } satisfies WalkedDirs));
+  } catch {
+    // A catalogue that will not keep them is a column drawn from the catalogue, as
+    // it was before.
+  }
+}
+
+// Every record of what a walk found, dropped: the catalogue that was just put
+// back is the one the reader asked for, and a number read off a disk the backup
+// does not describe is not a number to draw under it. Everything is forgotten
+// because a restore replaces the whole catalogue; what is left is a roll whose
+// next reading names it again.
+export async function forgetWalkedDirs(): Promise<void> {
+  try {
+    await ask(DIRS, 'readwrite', (s) => s.clear());
+  } catch {
+    // Nothing to forget.
+  }
+}
+
 // A handle kept in IndexedDB comes back without its permission: the browser
 // dropped it when the tab closed, and only the visitor can hand it back. Asking
 // costs one quiet prompt, which is why this runs at start-up rather than the
@@ -628,6 +827,11 @@ interface WalkSaved {
   // The frames the walk has found and the reading has not got to. The front of
   // this is the frame a reloaded reading picks up at.
   frames: string[];
+  // Written by the builds that kept only the front of the queue: the count of
+  // frames in hand when the position went down. `frames` was a slice of them, so
+  // a position with more in hand than it wrote cannot say where the rest were —
+  // see `loadWalk`. No reading of this build writes it.
+  unread?: number;
   // How many of the front of `frames` were already counted as read by the reading
   // that wrote this down: the rows are in hand and not in the catalogue yet, so
   // the next reading reads them again — and counts none of them twice.
@@ -646,13 +850,22 @@ async function loadWalk(folder: string): Promise<WalkSaved | null> {
     const fh = await walkFile(folder, false);
     if (!fh) return null;
     const saved = JSON.parse(await (await fh.getFile()).text()) as WalkSaved;
-    return Array.isArray(saved?.walked) &&
+    const ok = Array.isArray(saved?.walked) &&
       Array.isArray(saved?.pending) &&
       Array.isArray(saved?.frames) &&
       Number.isInteger(saved?.restored) &&
-      saved.progress
-      ? saved
-      : null;
+      saved.progress;
+    if (!ok) return null;
+    // The queue was cut off when this was written down by an older build: it knew
+    // where the first few hundred unread frames were and nothing of the rest, and
+    // a walk that takes it up walks the folders it lists as already walked — an
+    // update over a roll with a few thousand frames left in it reads the queue out
+    // and calls the roll read, never reaching the frames past the cut. A position
+    // that cannot say where the reading had got to is no position: the reading
+    // starts from the top and skips the frames it knows on their size and their
+    // time. A position this build wrote holds the whole queue and is always taken.
+    if (saved.unread !== undefined && saved.unread > saved.frames.length) return null;
+    return saved;
   } catch {
     return null;
   }
@@ -744,6 +957,12 @@ export async function scanFolder(
     }
   }
   const tree: Walk = { root: folder.handle, pending: [], walked: treeWalked };
+  // Whether this reading took up a position, read here and not after the walk:
+  // `tree.walked` is this same set and the walk fills it as it goes, so a reading
+  // that started from the top and one that started from a position look alike
+  // once either has been through the roll — and the difference is exactly what a
+  // reading may say about folders it never listed. See `prunePhotos`.
+  const resumed = treeWalked.size > 0;
   // A queue comes back as paths, so the folders are asked for again; a reading
   // with nothing written down starts at the folder it was pointed at — the
   // picked one, or the one the reader right-clicked inside it.
@@ -791,47 +1010,61 @@ export async function scanFolder(
   progress.at ??= '';
   const found = new Set(progress.dirs.map((d) => d.id));
   const normRoot = normPath(folder.name);
-  // The rows are counted from what the catalogue already holds, and the reading
-  // only adds to them. A number that started at nothing would sit under the
-  // catalogue's own for as long as the reading took to pass it, and the screen
-  // shows the larger of the two — so an update, where the reading is mostly
-  // frames the catalogue has, is a number that stands still and then jumps. What
-  // the reading adds is the frames it found that the catalogue does not hold yet:
-  // the number climbs for news alone, and ends on the true total.
-  const counts: Record<string, number> = {};
+  // The rows are counted from what the walk meets, and from nothing else. The
+  // catalogue used to be counted in first, so an update over a roll it already
+  // held wore the catalogue's own numbers for the whole of the reading — and a
+  // catalogue only ever grows: a frame taken off the disk is a row nothing takes
+  // out, so a folder the reader emptied kept the number it had, and every scan
+  // after that drew a count the disk no longer had. The walk meets every frame of
+  // the roll and is the only thing that knows what is there now; what it counted
+  // is what the column says. A reading that takes up a position starts from the
+  // numbers that position was written with: the folders it walked before this
+  // visit are folders it does not walk again, and a number that started at
+  // nothing would leave their frames out of the count for good.
+  const counts: Record<string, number> = { ...(saved?.progress.counts ?? {}) };
+  // A reading kept to one folder says nothing about the rest of the roll: its
+  // numbers drawn over the column would count a roll down to a branch of itself,
+  // and the branch below the folder it was pointed at is the whole of what it has
+  // walked. The row of the roll itself falls back on what the roll's own reading
+  // found, which is a number this one has no business standing under.
+  const fromPath = bounded ? normPath(from) : '';
+  const under = (path: string) => !bounded || path === fromPath || path.startsWith(`${fromPath}/`);
   const count = (rel: string, n = 1) => {
     const relNorm = rel.replace(/\\/g, '/');
     const cut = relNorm.lastIndexOf('/');
-    counts[normRoot] = (counts[normRoot] ?? 0) + n;
-    counts[normRoot.toLowerCase()] = counts[normRoot];
+    if (!bounded) {
+      counts[normRoot] = (counts[normRoot] ?? 0) + n;
+      counts[normRoot.toLowerCase()] = counts[normRoot];
+    }
     let path = '';
     for (const part of (cut < 0 ? '' : relNorm.slice(0, cut)).split('/')) {
       if (!part) continue;
       path = path ? `${path}/${part}` : part;
+      if (!under(path)) continue;
       const key = `${normRoot}/${path}`;
       counts[key] = (counts[key] ?? 0) + n;
       counts[key.toLowerCase()] = counts[key];
     }
   };
-  const relOfId = (id: string) => {
-    const normId = normPath(id);
-    const prefix = `${normRoot}/`.toLowerCase();
-    if (normId.toLowerCase().startsWith(prefix)) {
-      return normId.slice(normRoot.length + 1);
-    }
-    return normId;
-  };
-  for (const id of known.keys()) count(relOfId(id));
-  const knownPaths = new Set(
-    [...known.keys()].map((id) => relOfId(id))
-  );
+  // A frame the walk meets once is one frame, whether it is one the catalogue
+  // holds or one it has never seen: the walk lists every folder it walks into, so
+  // a name met twice is two entries for one frame — and both are counted once.
   const countedDiscovered = new Set<string>();
+  // The folders this reading could not open, and so has no word on: what they
+  // hold was never listed, and a reading that would repair the catalogue to the
+  // disk must not read silence for absence.
+  const failed = new Set<string>();
   const countDiscovered = (rel: string) => {
     const relClean = rel.replace(/\\/g, '/');
-    if (knownPaths.has(relClean) || countedDiscovered.has(relClean)) return;
-    countedDiscovered.add(relClean);
+    const key = relClean.toLowerCase();
+    if (countedDiscovered.has(key)) return;
+    countedDiscovered.add(key);
     count(relClean);
   };
+  // The frames a reading that was cut off had already counted come back with its
+  // position: the walk it takes up is the rest of the roll, and a frame it does
+  // not walk again is a frame it must not count again either.
+  for (const rel of saved?.frames ?? []) countedDiscovered.add(rel.replace(/\\/g, '/').toLowerCase());
   for (const e of entries) countDiscovered(e.rel);
   // The frames the catalogue already holds of the folder being read, which is
   // what the reading is measured against: a roll of a hundred thousand that has
@@ -840,8 +1073,8 @@ export async function scanFolder(
   // measured against the frames under that one alone.
   const heldPrefix = `${folder.name}/${from}`.toLowerCase();
   const held = bounded
-    ? [...known.keys()].filter((id) => id.toLowerCase().startsWith(heldPrefix)).length
-    : known.size;
+    ? rawKnown.filter((p) => p.id.toLowerCase().startsWith(heldPrefix)).length
+    : rawKnown.length;
   // What the screen is told, and how often. The column is redrawn out of the
   // count and the count moves with every frame, so telling the screen a frame at
   // a time is a whole tree rebuilt a frame at a time — on a roll of a hundred
@@ -866,6 +1099,9 @@ export async function scanFolder(
   // hands the list on with the rest of its position.
   const dirs: LibraryDir[] = progress.dirs;
   let batch: LibraryPhoto[] = [];
+  // When the batch last went down, for the clock above: a reading of a long roll is
+  // told its rows have landed at a beat the reader can watch, not in one lump.
+  let flushedAt = Date.now();
   // The frames this reading found for the first time, waiting to be handed on
   // with the batch they landed in. A frame the catalogue already had is not one
   // of these, however it came back — it is on the strip already, and a second
@@ -884,18 +1120,55 @@ export async function scanFolder(
   // string per batch, which is the page held still for the whole roll. A position
   // a step behind is a frame or two read again on the next visit, and a frame read
   // again is skipped on its size and its time.
+  //
+  // The whole queue and not the front of it: a position that kept a few hundred
+  // paths and dropped the rest could not say where the reading had got to, and a
+  // reload in the middle of a long roll either took that position up and called
+  // the roll read with thousands of frames still unread, or — as the guard in
+  // `loadWalk` had it — threw it away and left the roll unread with nothing to
+  // pick it up. A megabyte a second of string on the main thread is what the
+  // position costs; the alternative is the reading.
   let wroteAt = 0;
-  const write = async () => {
+  const write = async (force = false) => {
+    // A reading kept to one folder has no position to write: the file it would
+    // go in is the whole roll's, and a fraction of the tree written there is a
+    // roll that comes back with the rest of itself missing — the next update
+    // walks that one branch and calls the roll read.
+    if (bounded) return;
     const at = Date.now();
-    if (wroteAt !== 0 && at - wroteAt < WALK_MS) return;
+    // The clock is for the batches, which land faster than a position is worth
+    // writing. The two writes that bracket a pass are the reading's own shape and
+    // are not a matter of the clock: the one before it says the roll has not been
+    // listed yet, and a pass that is through in under `WALK_MS` — every roll of a
+    // few hundred folders — would leave that first one standing in for a walk that
+    // is over, so the next visit listed the whole tree again to learn what was
+    // already written down.
+    if (!force && wroteAt !== 0 && at - wroteAt < WALK_MS) return;
     wroteAt = at;
+    // The numbers go down with the position. `publish` is what the screen reads
+    // and it writes on a clock of its own; the count of a folder is only in
+    // `counts` and reaches `progress` through it, so a position written between
+    // two tells would say the reading had walked a folder and hold the number of
+    // it from before that folder — and a reading that takes the position up skips
+    // the folders it lists as walked, counting none of them again. The number a
+    // reloaded reading has for a folder it never walks again is this one.
+    progress.counts = { ...counts };
     const held = [...inHand, ...batch.map(pathOf)];
-    const savedEntries = entries.slice(0, 500).map((e) => e.rel);
+    // A folder this reading could not open is a folder it has no word on. Left
+    // among the walked, the position would say it had been read and the next
+    // reading — an update, which takes this position up — would skip it and never
+    // look inside it again: a number that went to nothing stays at nothing
+    // however many updates follow, and only a rescan, which throws the position
+    // away, brings it back. It goes back in the queue instead, where the next
+    // reading opens it and reads it as the folder it never managed to be.
+    const unsaid = failed.size ? new Set([...failed].map(normPath)) : null;
+    const walkedSaved = unsaid ? [...tree.walked].filter((r) => !unsaid.has(normPath(r))) : [...tree.walked];
+    const rest = tree.pending.map((p) => p.rel);
     await saveWalk(
       folder.name,
-      [...tree.walked],
-      tree.pending.map((p) => p.rel),
-      [...held, ...savedEntries],
+      walkedSaved,
+      unsaid ? [...unsaid, ...rest] : rest,
+      [...held, ...entries.map((e) => e.rel)],
       held.length,
       progress
     );
@@ -909,6 +1182,7 @@ export async function scanFolder(
     if (!batch.length) return;
     const rows = batch;
     batch = [];
+    flushedAt = Date.now();
     const db = await openDb();
     await new Promise<void>((resolve, reject) => {
       const hasThumbs = db.objectStoreNames.contains(THUMBS);
@@ -935,6 +1209,11 @@ export async function scanFolder(
     }
     await write();
   };
+  // Whether what is in hand goes down now: the batch is full, or the clock has come
+  // round. The frames of a roll smaller than `BATCH` never fill one, and waiting for
+  // the end of the reading to hand them over is the reading's length of an empty
+  // screen.
+  const due = () => batch.length > 0 && (batch.length >= BATCH || Date.now() - flushedAt >= FLUSH_MS);
 
   const readOne = async (handle: FileSystemFileHandle, rel: string): Promise<void> => {
     const file = await handle.getFile();
@@ -943,7 +1222,7 @@ export async function scanFolder(
     if (seen && seen.size === file.size && seen.mtime === file.lastModified) {
       if (!seen.handle) {
         batch.push({ ...seen, handle });
-        if (batch.length >= BATCH) await flush();
+        if (due()) await flush();
       }
       return;
     }
@@ -981,13 +1260,14 @@ export async function scanFolder(
     batch.push(row);
     if (!seen) {
       added.push(row);
-      // A frame the catalogue did not hold is news the rows have to show: it is
-      // the one thing an update adds to a number the catalogue already set.
-      count(rel);
+      // The row is not counted here. The walk counted every frame it found,
+      // this one included, the moment it found it — a second count on the way
+      // to the catalogue is the same frame twice, and a number that climbs past
+      // the roll and falls back to it when the reading ends.
     }
     progress.added++;
     inHand = inHand.filter((r) => r !== rel);
-    if (batch.length >= BATCH) await flush();
+    if (due()) await flush();
   };
 
   // The frames in hand, read a few at a time: reading a frame is almost all
@@ -1042,6 +1322,12 @@ export async function scanFolder(
 
   let stop = false;
   let walked = false;
+  // The position before a single folder has been listed: a roll whose first pass
+  // takes a minute to get through the folders at its top is a roll with nothing
+  // written down for the whole of that minute, and a reload in it found nothing
+  // to pick up and left the reading where it was — started again by hand. What
+  // this writes down is the folder the reading was pointed at and no frame of it.
+  await write();
   // One bounded pass at a time, and the pass yields between frames, so a folder
   // clicked while the scan runs is read on the next jump rather than after the
   // whole tree. Nothing clicked: the picked folder and its frames, then each
@@ -1069,6 +1355,9 @@ export async function scanFolder(
         },
         (file) => {
           countDiscovered(file.rel);
+        },
+        (rel) => {
+          failed.add(rel);
         }
       );
       // The names come in before the frames they hold: the column grows one pass
@@ -1087,8 +1376,9 @@ export async function scanFolder(
       // The position before a frame of this pass is looked at: a reload here
       // reads the pass's frames again, and a frame already in the catalogue is
       // skipped anyway — where the other order would leave the last pass's
-      // frames out of the catalogue until the next visit.
-      await write();
+      // frames out of the catalogue until the next visit. Written whatever the
+      // clock says: it is the pass that is over, not a batch.
+      await write(true);
       // A pass is worth telling the screen about whether the clock says so or
       // not: the names of a whole layer have just arrived, and the column is
       // built from them.
@@ -1103,7 +1393,27 @@ export async function scanFolder(
   // walks the roll from the top and skips what has not moved. A reading kept to
   // one folder never wrote one down, and the file it would clear is the whole
   // roll's — a position another reading is in the middle of.
-  if (!stop) await clearWalk(folder.name);
+  // The roll's own position, and only the reading of the whole roll has one to
+  // clear: a reading kept to a subfolder never wrote it, and clearing it here
+  // throws away the rest of a roll an earlier reading stopped in the middle of.
+  if (!stop && !bounded) await clearWalk(folder.name);
+  // The catalogue brought to the disk. What the reading met is what is there, but
+  // only for a reading that met the whole of what it was pointed at: one that was
+  // stopped, one that took up a position — the folders that position had already
+  // walked are folders this reading never listed — and one that met a folder it
+  // could not open are all readings with no word on the frames they did not
+  // reach, and what they did not reach is not theirs to drop. See `prunePhotos`.
+  if (!stop && !failed.size && (bounded || !resumed)) {
+    await prunePhotos(folder.name, bounded ? from : '', countedDiscovered);
+  }
+  // What the walk found, kept where the catalogue is kept, so the column has it
+  // on the next visit whether the reading ran to its end or was cut off. A
+  // reading of the whole roll that reached its end is the roll and replaces what
+  // was there — a folder it did not meet is a folder that is gone, and its number
+  // with it. A reading kept to one folder knows one branch of the roll and is
+  // folded in; a reading that was stopped keeps its numbers for the folders it
+  // walked and none for the rest, which is what it counted. See `saveWalkedDirs`.
+  await saveWalkedDirs(folder.name, dirs, stop ? null : counts, !stop && !bounded, bounded ? from : '', failed);
   // The last word, whatever the clock says: the frames the reading got to are
   // the frames the column is about to be told about one last time.
   publish(true);

docker/frontend/src/engine/libraryBackup.ts

@@ -23,6 +23,7 @@
 import type { LibraryEdit, LibraryFolder } from './library';
 import {
   ensurePermission,
+  forgetWalkedDirs,
   getPhotoThumb,
   listEdits,
   listFolders,
@@ -396,6 +397,10 @@ export async function restoreNow(onProgress?: (p: BackupProgress) => void): Prom
 
   // Instant catalog metadata & edits restoration (< 100ms)
   const photos = await restoreCatalogue(photoList, editList);
+  // And the walks are forgotten with it: the numbers a walk wrote down are off a
+  // disk this backup does not describe, and a column drawn under them would say a
+  // folder holds what it held the last time the reader looked at it.
+  await forgetWalkedDirs();
 
   // Activate backup tile fetcher for lazy rendering on screen
   setBackupTileFetcher(readBackupTile);

docker/frontend/src/engine/rawDevelop.ts

@@ -567,21 +567,57 @@ function getCamToSrgbMatrix(cd: any, brand?: CameraBrand): number[][] {
   }
 }
 
-export async function developRaw(bytes: Uint8Array, fileName?: string): Promise<Uint8Array> {
-  // 1. Fast extraction of the embedded JPEG preview.
-  // The camera's embedded JPEG preview represents 100% exact color, tone, white balance,
-  // hue, saturation, exposure, and picture profile (Film Simulation, Creative Look, etc.)
-  // generated directly by the camera hardware (Fujifilm, Sony, Olympus, Nikon, Canon,
-  // Ricoh, Lumix, Leica, iPhone ProRAW, DNG).
-  const preview = extractEmbeddedJpeg(bytes);
-  if (preview && preview.length > 10000) {
-    return preview;
+// The size a JPEG declares for itself, off its SOF segment — a header read, no
+// decoder. Null when the bytes carry no readable frame header.
+export function jpegFrameSize(jpeg: Uint8Array): { width: number; height: number } | null {
+  for (let i = 2; i + 9 < jpeg.length; ) {
+    if (jpeg[i] !== 0xff) {
+      i++;
+      continue;
+    }
+    const marker = jpeg[i + 1];
+    if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) {
+      return { height: (jpeg[i + 5] << 8) | jpeg[i + 6], width: (jpeg[i + 7] << 8) | jpeg[i + 8] };
+    }
+    i += 2 + ((jpeg[i + 2] << 8) | jpeg[i + 3]);
   }
+  return null;
+}
+
+// Is the embedded preview the frame itself? Bodies that write one at the frame
+// size (Nikon NEF 6016x4016 out of 6032x4032, Sony ARW, an iPhone ProRAW DNG)
+// differ from the sensor by a margin row or two; a thumbnail out of the same
+// sensor is 1920x1280, a third of the long side, so 90% divides them cleanly.
+// Unreadable sizes keep the old behaviour and trust the preview.
+function previewIsFrame(preview: Uint8Array, meta: any): boolean {
+  const size = jpegFrameSize(preview);
+  const frame = Math.max(
+    Number(meta?.iwidth) || 0,
+    Number(meta?.iheight) || 0,
+    Number(meta?.width) || 0,
+    Number(meta?.height) || 0
+  );
+  if (!size || !frame) return true;
+  return Math.max(size.width, size.height) >= frame * 0.9;
+}
+
+export async function developRaw(bytes: Uint8Array, fileName?: string): Promise<Uint8Array> {
+  // The embedded JPEG is the camera's own rendering — exact colour, tone, white
+  // balance and picture profile, generated by the body itself. It is the frame
+  // only when it is the size of the frame; on most bodies it is a thumbnail
+  // (1920x1280 out of a 24MP sensor, measured on a Fuji RAF and a Lumix RW2), so
+  // handing it back is what opened a 24MP file as a 1920 wide one. Whether it is
+  // the frame is known from the sensor size, which costs one LibRaw open (91ms
+  // measured, metadata only — no demosaic) against seconds for the develop.
+  const preview = extractEmbeddedJpeg(bytes);
 
   const raw = new LibRaw();
   try {
     await raw.open(bytes.slice() as unknown as BufferSource, SETTINGS);
     const meta = await raw.metadata(true);
+    if (preview && preview.length > 10000 && previewIsFrame(preview, meta)) {
+      return preview;
+    }
     const img = await raw.imageData();
     const cd = meta?.color_data;
     if (!img || (!cd?.cam_mul && !cd?.pre_mul) || (!cd.rgb_cam && !cd.cam_xyz)) {
@@ -657,7 +693,7 @@ export async function developRaw(bytes: Uint8Array, fileName?: string): Promise<
     if (!first) throw new Error('no surface for the develop');
 
     let finalShot = first;
-    let refJpeg = extractEmbeddedJpeg(bytes);
+    let refJpeg = preview;
     if (!refJpeg) {
       refJpeg = await cameraPreview(raw).catch(() => null);
     }
@@ -691,6 +727,9 @@ export async function developRaw(bytes: Uint8Array, fileName?: string): Promise<
     if (!jpeg?.length) throw new Error('develop produced no bytes');
     return jpeg;
   } catch (err) {
+    // The preview is also the fallback, as it always was: a file whose sensor
+    // will not decode still opens as the camera rendered it.
+    if (preview && preview.length > 10000) return preview;
     const fallback = await rawThumbnail(bytes).catch(() => null);
     if (fallback) return fallback;
     throw err;

docker/frontend/src/engine/rollWalk.ts

@@ -89,7 +89,8 @@ export async function walkPass(
   keep: (name: string) => boolean,
   jump: () => string | null,
   onDir?: (rel: string) => void,
-  onFile?: (file: WalkFile) => void
+  onFile?: (file: WalkFile) => void,
+  onFail?: (rel: string) => void
 ): Promise<boolean> {
   for (let i = 0; i < DIRS_PER_PASS && walk.pending.length; i++) {
     await jumpQueue(walk, jump());
@@ -115,6 +116,10 @@ export async function walkPass(
       }
     } catch {
       // A subfolder that will not open is skipped; the rest of the roll stands.
+      // It is also a folder this walk has no word on — what it holds was never
+      // listed, so a caller about to take the walk's word for what the disk
+      // holds is told here that this folder is not part of it.
+      onFail?.(here.rel);
     }
   }
   return walk.pending.length === 0;