web: walk a roll layer by layer, and let the strip drop the branch

2026-09-28 19:55:02 +07:00
parent bd206a8be9
commit d0fddba1e8
7 changed files with 282 additions and 63 deletions
@@ -15,6 +15,7 @@
 import type { Recipe } from '../../shared/types';
 import { readCapturedAt } from './imageOps';
 import { isRawName, rawThumbnail } from './rawDevelop';
+import { walkPass, type WalkFile } from './rollWalk';

 const DB_NAME = 'recipescam-library';
 const FOLDERS = 'folders';
@@ -235,32 +236,13 @@ export interface ScanProgress {
  added: number;
 }

-// The walk goes all the way down: a picked folder is usually a roll of
-// date-named subfolders, and a catalogue that only saw its top level would be
-// empty. Names beginning with `.` or `@` are skipped — that is `.thumbnails`,
-// `.git`, and the `@eaDir` a Synology writes beside every frame.
-async function collectFiles(
-  dir: FileSystemDirectoryHandle,
-  prefix: string,
-  out: { handle: FileSystemFileHandle; rel: string }[]
-): Promise<void> {
-  for await (const entry of dir.values()) {
-    if (/^[.@]/.test(entry.name)) continue;
-    if (entry.kind === 'directory') {
-      try {
-        await collectFiles(entry as FileSystemDirectoryHandle, `${prefix}${entry.name}/`, out);
-      } catch {
-        // A subfolder that will not open is skipped; the rest of the roll stands.
-      }
-    } else if (isSupportedPhoto(entry.name)) {
-      out.push({ handle: entry as FileSystemFileHandle, rel: `${prefix}${entry.name}` });
-    }
-  }
-}
-
 // Walk the folder, keep what is new or changed, and leave the rest alone: a
 // second scan of a 2000-file folder only reads the frames that moved.
 //
+// The walk itself lives in `rollWalk`: layer by layer from the picked folder
+// down, one bounded pass at a time, with the folder the reader just clicked
+// pulled to the front of the queue. `jump` is that click.
+//
 // ponytail: read one frame at a time, whole, on the main thread — a RAW is read
 // as its bytes for LibRaw and released again. A folder of 5000 RAW files takes
 // minutes and makes the page lumpy while it runs. Move the walk into a worker
@@ -268,12 +250,13 @@ async function collectFiles(
 export async function scanFolder(
  folder: LibraryFolder,
  onProgress?: (p: ScanProgress) => void,
-  shouldStop?: () => boolean
+  shouldStop?: () => boolean,
+  jump?: () => string | null
 ): Promise<ScanProgress> {
  const known = new Map((await listPhotos(folder.name)).map((p) => [p.id, p]));
-  const entries: { handle: FileSystemFileHandle; rel: string }[] = [];
-  await collectFiles(folder.handle, '', entries);
-  const progress: ScanProgress = { folder: folder.name, total: entries.length, done: 0, added: 0 };
+  const pending = [{ dir: folder.handle, rel: '' }];
+  const entries: WalkFile[] = [];
+  const progress: ScanProgress = { folder: folder.name, total: 0, done: 0, added: 0 };
  let batch: LibraryPhoto[] = [];
  // One transaction per batch, a put per row: a store with `keyPath: 'id'` takes
  // a record, not an array of them.
@@ -290,39 +273,53 @@ export async function scanFolder(
      tx.onerror = () => reject(tx.error);
    });
  };
-  for (const { handle, rel } of entries) {
-    if (shouldStop?.()) break;
-    progress.done++;
-    try {
-      const file = await handle.getFile();
-      const id = photoId(folder.name, rel);
-      const seen = known.get(id);
-      if (seen && seen.size === file.size && seen.taken && seen.taken === file.lastModified && seen.thumb) {
-        onProgress?.(progress);
-        continue;
+  let stop = false;
+  let walked = false;
+  // One bounded pass at a time, and the pass yields between frames, so a folder
+  // clicked while the scan runs is walked on the next jump rather than after the
+  // whole tree. Nothing clicked: parents first, then each layer below, deepest
+  // last — the order `pending` is filled in.
+  do {
+    const from = entries.length;
+    walked = await walkPass(pending, entries, isSupportedPhoto, jump ?? (() => null));
+    progress.total += entries.length - from;
+    for (const { handle, rel } of entries.slice(from)) {
+      if (shouldStop?.()) {
+        stop = true;
+        break;
      }
-      const bytes = new Uint8Array(await file.arrayBuffer());
-      const [thumb, taken] = [await makeThumbnail(file, bytes), (await readCapturedAt(bytes)) ?? file.lastModified];
-      const cut = rel.lastIndexOf('/');
-      batch.push({
-        id,
-        folder: folder.name,
-        dir: cut < 0 ? '' : rel.slice(0, cut),
-        name: handle.name,
-        handle,
-        thumb,
-        taken,
-        size: file.size,
-        addedAt: seen?.addedAt ?? Date.now(),
-      });
-      progress.added++;
-      if (batch.length >= BATCH) await flush();
-    } catch {
-      // A frame that will not read is a frame the catalogue skips: one bad file
-      // in a folder is not a failed folder.
+      progress.done++;
+      try {
+        const file = await handle.getFile();
+        const id = photoId(folder.name, rel);
+        const seen = known.get(id);
+        if (seen && seen.size === file.size && seen.taken && seen.taken === file.lastModified && seen.thumb) {
+          onProgress?.(progress);
+          continue;
+        }
+        const bytes = new Uint8Array(await file.arrayBuffer());
+        const [thumb, taken] = [await makeThumbnail(file, bytes), (await readCapturedAt(bytes)) ?? file.lastModified];
+        const cut = rel.lastIndexOf('/');
+        batch.push({
+          id,
+          folder: folder.name,
+          dir: cut < 0 ? '' : rel.slice(0, cut),
+          name: handle.name,
+          handle,
+          thumb,
+          taken,
+          size: file.size,
+          addedAt: seen?.addedAt ?? Date.now(),
+        });
+        progress.added++;
+        if (batch.length >= BATCH) await flush();
+      } catch {
+        // A frame that will not read is a frame the catalogue skips: one bad file
+        // in a folder is not a failed folder.
+      }
+      onProgress?.(progress);
    }
-    onProgress?.(progress);
-  }
+  } while (!walked && !stop);
  await flush();
  return progress;
 }
@@ -0,0 +1,68 @@
+// Walking a picked folder is the one part of the catalogue that can run for
+// minutes: a roll is dated folders inside dated folders, and a library can hold
+// thousands. The walk goes from the parent down, one layer at a time, so the
+// frames nearest the top are indexed first — and a folder the reader clicks is
+// put in front of the queue, so the branch in front of them is read before the
+// layer it sits in would have come round. A pass covers `DIRS_PER_PASS`
+// directories and then hands the thread back, which is what keeps the page
+// answering while a scan runs.
+//
+// Pure on purpose: it only reads `values()` off a directory handle, so a check
+// can walk a stand-in tree without a browser (scripts/roll-walk-check.mjs).
+
+export interface WalkFile {
+  handle: FileSystemFileHandle;
+  rel: string;
+}
+
+interface Pending {
+  dir: FileSystemDirectoryHandle;
+  rel: string;
+}
+
+// Directories one pass walks before it yields: small enough that a click is
+// answered within a second, large enough that a folder of a few hundred is one
+// pass rather than a hundred.
+export const DIRS_PER_PASS = 200;
+
+// The folder the reader clicked goes first. `rel` is how the queue spells a
+// folder — '' for the picked one, `2026/04/` under it — and a folder already
+// walked is simply not in the queue, which is the no-op case.
+//
+// ponytail: only a folder the queue already holds can jump, which is every
+// folder the tree can draw, because a row exists per indexed frame. A tree that
+// listed folders before their first frame would need `getDirectoryHandle` to
+// descend to the clicked path here.
+export function jumpQueue(pending: Pending[], rel: string | null): void {
+  if (rel === null) return;
+  const at = pending.findIndex((p) => p.rel === rel);
+  if (at > 0) pending.unshift(pending.splice(at, 1)[0]);
+}
+
+// One pass over the queue. The queue is the walk's own state: what is behind has
+// been walked, what is ahead is in the order it was found — parents before
+// children, shallower layers before deeper ones. Returns true when nothing is
+// left to walk.
+export async function walkPass(
+  pending: Pending[],
+  out: WalkFile[],
+  keep: (name: string) => boolean,
+  jump: () => string | null
+): Promise<boolean> {
+  for (let i = 0; i < DIRS_PER_PASS && pending.length; i++) {
+    jumpQueue(pending, jump());
+    const here = pending.shift()!;
+    try {
+      for await (const entry of here.dir.values()) {
+        // Names beginning with `.` or `@` are skipped — that is `.thumbnails`,
+        // `.git`, and the `@eaDir` a Synology writes beside every frame.
+        if (/^[.@]/.test(entry.name)) continue;
+        if (entry.kind === 'directory') pending.push({ dir: entry as FileSystemDirectoryHandle, rel: `${here.rel}${entry.name}/` });
+        else if (keep(entry.name)) out.push({ handle: entry as FileSystemFileHandle, rel: `${here.rel}${entry.name}` });
+      }
+    } catch {
+      // A subfolder that will not open is skipped; the rest of the roll stands.
+    }
+  }
+  return pending.length === 0;
+}