fix(library): a catalogue is read without the file handles it was keeping

A row of `photos` kept the frame's own FileSystemFileHandle, and reading the
catalogue handed them all to the page at once. Measured on a catalogue of
504290 frames, 458103 of them with a handle, opening the library left the JS
heap at 133MB while the browser process held 5.7GB and the renderer 2.1GB:
handles are not the page's heap, so nothing ever asked the collector to run,
and the rescan's own reload of the catalogue every thirty seconds piled one
such reading on the next until the tab was out of memory.

No row keeps a handle now. The catalogue is read a few thousand rows at a time
— the four and a half gigabytes of live handles came from asking for all of
them at once — and a frame's file is asked of its folder's handle when it is
drawn or opened, `fileHandleFor`. `reconnectPhotosForFolders` no longer writes
handles back into the frames, which is what put them there.

504290 rows read in chunks peak at 1.4GB and settle at 1.1GB, against 8.0GB in
one read before.

docker/frontend/src/Library.tsx

@@ -3432,7 +3432,9 @@ function useThumbUrls(photos: LibraryPhoto[], also?: LibraryPhoto | null, stamp?
         setUrls((prev) => ({ ...prev, ...landed }));
       }
 
-      const queue = list.filter((p) => !(p.id in landed) && p.handle);
+      // A frame off the disk is one no server owns: its file is asked of the
+      // folder's handle when the tile is made, so there is no handle to test for.
+      const queue = list.filter((p) => !(p.id in landed) && !p.remoteId);
       await Promise.all(
         Array.from({ length: Math.min(WALL_LANES, queue.length) }, async () => {
           while (alive && queue.length) {

docker/frontend/src/engine/library.ts

@@ -4,9 +4,10 @@
 //
 // The three stores are what the folder is made of: `folders` keeps the directory
 // handle so the catalogue survives a reload without a second trip through the
-// picker, `photos` keeps a file handle (the RAW itself is read only when a frame
+// picker, `photos` keeps what was found there — the RAW itself is read only when
-// is opened) plus the small JPEG the grid paints, and `edits` keeps the recipe a
+// a frame is opened, and the file it is read from is asked of the folder's own
-// frame was left at, keyed by the same id the grid uses.
+// handle again, see `fileHandleFor` — and `edits` keeps the recipe a frame was
+// left at, keyed by the same id the grid uses.
 //
 // ponytail: Dexie would wrap this in three lines, but IndexedDB stores a
 // FileSystemHandle by itself and the whole surface is seven calls — a dependency
@@ -62,6 +63,13 @@ const WALK_MS = 1000;
 // time drawing itself. Five times a second is a counter that moves to the eye
 // and a page that is doing the reading instead.
 const TELL_MS = 200;
+// Rows of the catalogue read at once. A row carries a file handle, and a handle
+// is not the page's memory to measure: read whole, a catalogue of half a million
+// frames leaves the JS heap at 133MB and the browser process holding 5.7GB it
+// only hands back when the collector happens to run. A few thousand rows at a
+// time is a heap that does move, and the handles of one chunk are gone before the
+// next is asked for.
+const READ_CHUNK = 4000;
 
 // One LibRaw open at a time, whoever asks for it. A RAW is opened inside a worker
 // the library builds with a quarter of a gigabyte of linear memory of its own
@@ -368,13 +376,37 @@ export async function listFolders(): Promise<LibraryFolder[]> {
   }
 }
 
-export async function reconnectPhotosForFolders(
+// A frame's file handle, off the folder it was found in. No row of the catalogue
-  newFolderHandle: FileSystemDirectoryHandle
+// keeps one — a catalogue of half a million rows keeping one apiece is what took
-): Promise<{ reconnectedFolders: number; reconnectedPhotos: number }> {
+// the screen down, and it is the folder's own handle, one row, that is needed to
+// ask for it again. The folder part of a path is opened once and kept: a wall's
+// worth of frames is nearly always a handful of folders.
+const dirHandles = new Map<string, FileSystemDirectoryHandle | null>();
+
+export async function fileHandleFor(photo: LibraryPhoto): Promise<FileSystemFileHandle | null> {
+  if (photo.handle) return photo.handle;
+  if (photo.remoteId) return null;
+  const folder = (await listFolders()).find((f) => f.handle && normPath(f.name) === normPath(photo.folder));
+  if (!folder?.handle) return null;
+  const key = `${folder.name}\u0000${photo.dir}`;
+  if (!dirHandles.has(key)) dirHandles.set(key, photo.dir ? await openAt(folder.handle, photo.dir) : folder.handle);
+  const dir = dirHandles.get(key);
+  if (!dir) return null;
+  try {
+    return await dir.getFileHandle(photo.name);
+  } catch {
+    return null;
+  }
+}
+
+// A folder the reader has just handed back, and what the catalogue knows of the
+// folders under it: the record goes in again, so a visit that follows finds the
+// folder it was pointed at without a second trip through the picker. Nothing is
+// written to the frames here — a frame's file is asked of this folder's handle
+// when it is drawn, by `fileHandleFor`, so there is nothing to hand back to it.
+export async function reconnectPhotosForFolders(newFolderHandle: FileSystemDirectoryHandle): Promise<void> {
   const existingFolders = await listFolders();
   const allPhotos = (await readPhotos()) ?? [];
-  let reconnectedFolders = 0;
-  let reconnectedPhotos = 0;
 
   const folderTargets: { name: string; handle: FileSystemDirectoryHandle }[] = [];
   folderTargets.push({ name: newFolderHandle.name, handle: newFolderHandle });
@@ -411,54 +443,10 @@ export async function reconnectPhotosForFolders(
     const labels = new Map(existingFolders.map((f) => [f.name, f.label]));
     for (const target of folderTargets) {
       store.put({ name: target.name, label: labels.get(target.name), handle: target.handle });
-      reconnectedFolders++;
     }
     tx.oncomplete = () => resolve();
     tx.onerror = () => reject(tx.error);
   });
-
-  const photosToUpdate: LibraryPhoto[] = [];
-  for (const target of folderTargets) {
-    const matchingPhotos = allPhotos.filter(
-      (p) => normPath(p.folder) === normPath(target.name) && !p.handle && !p.remoteId
-    );
-    if (!matchingPhotos.length) continue;
-
-    const dirCache = new Map<string, FileSystemDirectoryHandle>();
-    dirCache.set('', target.handle);
-
-    for (const p of matchingPhotos) {
-      try {
-        let dirHandle = dirCache.get(p.dir);
-        if (!dirHandle) {
-          const opened = await openAt(target.handle, p.dir);
-          if (!opened) continue;
-          dirCache.set(p.dir, opened);
-          dirHandle = opened;
-        }
-        const fileHandle = await dirHandle.getFileHandle(p.name);
-        if (fileHandle) {
-          p.handle = fileHandle;
-          photosToUpdate.push(p);
-          reconnectedPhotos++;
-        }
-      } catch {}
-    }
-  }
-
-  if (photosToUpdate.length) {
-    await new Promise<void>((resolve, reject) => {
-      const tx = db.transaction(PHOTOS, 'readwrite');
-      const store = tx.objectStore(PHOTOS);
-      for (const p of photosToUpdate) {
-        store.put(p);
-      }
-      tx.oncomplete = () => resolve();
-      tx.onerror = () => reject(tx.error);
-    });
-  }
-
-  return { reconnectedFolders, reconnectedPhotos };
 }
 
 // The picker's own dialog; `id` makes the browser reopen at the folder this page
@@ -1350,13 +1338,7 @@ export async function scanFolder(
     const file = await withTimeout(handle.getFile(), READ_MS);
     const id = photoId(folder.name, rel);
     const seen = known.get(id);
-    if (seen && seen.size === file.size && seen.mtime === file.lastModified) {
+    if (seen && seen.size === file.size && seen.mtime === file.lastModified) return;
-      if (!seen.handle) {
-        batch.push({ ...seen, handle });
-        if (due()) await flush();
-      }
-      return;
-    }
     // Where the frame was taken, off the few hundred kilobytes of its head that
     // its own EXIF lives in — where the 32MB a camera's RAW used to be read for
     // bought a preview this reading no longer wants. Nothing here looks at a
@@ -1380,7 +1362,6 @@ export async function scanFolder(
       folder: normPath(folder.name),
       dir: cut < 0 ? '' : normPath(relNorm.slice(0, cut)),
       name: handle.name,
-      handle,
       taken: taken ?? file.lastModified,
       size: file.size,
       mtime: file.lastModified,
@@ -1948,6 +1929,7 @@ export function cachedPhotos(): LibraryPhoto[] {
 
 export function clearLibraryCache(): void {
   cached = null;
+  dirHandles.clear();
 }
 
 let backupTileFetcher: ((id: string) => Promise<Blob | null>) | null = null;
@@ -2167,7 +2149,7 @@ async function makeOneThumb(photo: LibraryPhoto): Promise<Blob | null> {
   // A server's frame has no file to open: its tile is the one Immich drew, asked
   // for here and kept, so the second look comes out of the store and the sync's
   // own download is only paid once.
-  if (!photo.handle && photo.remoteId) {
+  if (photo.remoteId) {
     try {
       const res = await fetch(remoteUrl('thumbnail', photo));
       if (!res.ok) return null;
@@ -2178,7 +2160,6 @@ async function makeOneThumb(photo: LibraryPhoto): Promise<Blob | null> {
       return null;
     }
   }
-  if (!photo.handle) return null;
   try {
     const file = await readPhotoFile(photo);
     const name = file.name;
@@ -2298,12 +2279,32 @@ export async function readPhotos(folder?: string): Promise<LibraryPhoto[] | null
     // spells it `''`. Asking the index for the raw name is asking for a spelling
     // nothing is filed under — the catalogue comes back empty, every frame of the
     // roll is read as new, and a second reading of a drive it already holds drags
-    // a handle for every frame of it into memory at once. Ask for what is filed.
+    // a handle for every frame of it into memory at once. An id is `folder/…`, so
-    const rows = await ask<LibraryPhoto[]>(PHOTOS, 'readonly', (s) =>
+    // one folder's rows are the run of keys between the folder and its own name:
-      folder ? s.index('folder').getAll(normPath(folder)) : s.getAll()
+    // the rows the index handed back, asked of the store, a chunk at a time.
-    );
+    const root = folder === undefined ? null : normPath(folder);
-    for (let i = 0; i < rows.length; i++) {
+    const at = (last: string | null): IDBKeyRange =>
-      if (rows[i].thumb) delete (rows[i] as { thumb?: Blob | null }).thumb;
+      root === null
+        ? IDBKeyRange.lowerBound(last ?? '', last !== null)
+        : IDBKeyRange.bound(last ?? `${root}/`, `${root}/\uffff`, last !== null, false);
+    const rows: LibraryPhoto[] = [];
+    let last: string | null = null;
+    for (;;) {
+      const chunk = await new Promise<LibraryPhoto[]>((resolve, reject) => {
+        const req = db.transaction(PHOTOS, 'readonly').objectStore(PHOTOS).getAll(at(last), READ_CHUNK);
+        req.onsuccess = () => resolve(req.result as LibraryPhoto[]);
+        req.onerror = () => reject(req.error);
+      });
+      if (!chunk.length) break;
+      last = chunk[chunk.length - 1].id;
+      for (const row of chunk) {
+        // Neither is read again: the file a frame is opened from is the folder's
+        // business — see `fileHandleFor` — and the tile is the store's own.
+        delete (row as { handle?: unknown }).handle;
+        delete (row as { thumb?: unknown }).thumb;
+      }
+      rows.push(...chunk);
+      if (chunk.length < READ_CHUNK) break;
     }
     rows.sort((a, b) => b.taken - a.taken);
     if (!folder) cached = rows;
@@ -2381,8 +2382,9 @@ export async function readPhotoFile(photo: LibraryPhoto): Promise<File> {
     const blob = await res.blob();
     return new File([blob], photo.name, { type: blob.type || 'application/octet-stream' });
   }
-  if (!photo.handle) throw new Error('no-file');
+  const handle = await fileHandleFor(photo);
-  return photo.handle.getFile();
+  if (!handle) throw new Error('no-file');
+  return handle.getFile();
 }
 
 // --- edits -----------------------------------------------------------------