diff --git a/docker/frontend/src/Library.tsx b/docker/frontend/src/Library.tsx index 7c4953c..af593f0 100644 --- a/docker/frontend/src/Library.tsx +++ b/docker/frontend/src/Library.tsx @@ -3432,7 +3432,9 @@ function useThumbUrls(photos: LibraryPhoto[], also?: LibraryPhoto | null, stamp? setUrls((prev) => ({ ...prev, ...landed })); } - const queue = list.filter((p) => !(p.id in landed) && p.handle); + // A frame off the disk is one no server owns: its file is asked of the + // folder's handle when the tile is made, so there is no handle to test for. + const queue = list.filter((p) => !(p.id in landed) && !p.remoteId); await Promise.all( Array.from({ length: Math.min(WALL_LANES, queue.length) }, async () => { while (alive && queue.length) { diff --git a/docker/frontend/src/engine/library.ts b/docker/frontend/src/engine/library.ts index 468f435..79f3ef1 100644 --- a/docker/frontend/src/engine/library.ts +++ b/docker/frontend/src/engine/library.ts @@ -4,9 +4,10 @@ // // The three stores are what the folder is made of: `folders` keeps the directory // handle so the catalogue survives a reload without a second trip through the -// picker, `photos` keeps a file handle (the RAW itself is read only when a frame -// is opened) plus the small JPEG the grid paints, and `edits` keeps the recipe a -// frame was left at, keyed by the same id the grid uses. +// picker, `photos` keeps what was found there — the RAW itself is read only when +// a frame is opened, and the file it is read from is asked of the folder's own +// handle again, see `fileHandleFor` — and `edits` keeps the recipe a frame was +// left at, keyed by the same id the grid uses. // // ponytail: Dexie would wrap this in three lines, but IndexedDB stores a // FileSystemHandle by itself and the whole surface is seven calls — a dependency @@ -62,6 +63,13 @@ const WALK_MS = 1000; // time drawing itself. Five times a second is a counter that moves to the eye // and a page that is doing the reading instead. const TELL_MS = 200; +// Rows of the catalogue read at once. A row carries a file handle, and a handle +// is not the page's memory to measure: read whole, a catalogue of half a million +// frames leaves the JS heap at 133MB and the browser process holding 5.7GB it +// only hands back when the collector happens to run. A few thousand rows at a +// time is a heap that does move, and the handles of one chunk are gone before the +// next is asked for. +const READ_CHUNK = 4000; // One LibRaw open at a time, whoever asks for it. A RAW is opened inside a worker // the library builds with a quarter of a gigabyte of linear memory of its own @@ -368,13 +376,37 @@ export async function listFolders(): Promise { } } -export async function reconnectPhotosForFolders( - newFolderHandle: FileSystemDirectoryHandle -): Promise<{ reconnectedFolders: number; reconnectedPhotos: number }> { +// A frame's file handle, off the folder it was found in. No row of the catalogue +// keeps one — a catalogue of half a million rows keeping one apiece is what took +// the screen down, and it is the folder's own handle, one row, that is needed to +// ask for it again. The folder part of a path is opened once and kept: a wall's +// worth of frames is nearly always a handful of folders. +const dirHandles = new Map(); + +export async function fileHandleFor(photo: LibraryPhoto): Promise { + if (photo.handle) return photo.handle; + if (photo.remoteId) return null; + const folder = (await listFolders()).find((f) => f.handle && normPath(f.name) === normPath(photo.folder)); + if (!folder?.handle) return null; + const key = `${folder.name}\u0000${photo.dir}`; + if (!dirHandles.has(key)) dirHandles.set(key, photo.dir ? await openAt(folder.handle, photo.dir) : folder.handle); + const dir = dirHandles.get(key); + if (!dir) return null; + try { + return await dir.getFileHandle(photo.name); + } catch { + return null; + } +} + +// A folder the reader has just handed back, and what the catalogue knows of the +// folders under it: the record goes in again, so a visit that follows finds the +// folder it was pointed at without a second trip through the picker. Nothing is +// written to the frames here — a frame's file is asked of this folder's handle +// when it is drawn, by `fileHandleFor`, so there is nothing to hand back to it. +export async function reconnectPhotosForFolders(newFolderHandle: FileSystemDirectoryHandle): Promise { const existingFolders = await listFolders(); const allPhotos = (await readPhotos()) ?? []; - let reconnectedFolders = 0; - let reconnectedPhotos = 0; const folderTargets: { name: string; handle: FileSystemDirectoryHandle }[] = []; folderTargets.push({ name: newFolderHandle.name, handle: newFolderHandle }); @@ -411,54 +443,10 @@ export async function reconnectPhotosForFolders( const labels = new Map(existingFolders.map((f) => [f.name, f.label])); for (const target of folderTargets) { store.put({ name: target.name, label: labels.get(target.name), handle: target.handle }); - reconnectedFolders++; } tx.oncomplete = () => resolve(); tx.onerror = () => reject(tx.error); }); - - const photosToUpdate: LibraryPhoto[] = []; - for (const target of folderTargets) { - const matchingPhotos = allPhotos.filter( - (p) => normPath(p.folder) === normPath(target.name) && !p.handle && !p.remoteId - ); - if (!matchingPhotos.length) continue; - - const dirCache = new Map(); - dirCache.set('', target.handle); - - for (const p of matchingPhotos) { - try { - let dirHandle = dirCache.get(p.dir); - if (!dirHandle) { - const opened = await openAt(target.handle, p.dir); - if (!opened) continue; - dirCache.set(p.dir, opened); - dirHandle = opened; - } - const fileHandle = await dirHandle.getFileHandle(p.name); - if (fileHandle) { - p.handle = fileHandle; - photosToUpdate.push(p); - reconnectedPhotos++; - } - } catch {} - } - } - - if (photosToUpdate.length) { - await new Promise((resolve, reject) => { - const tx = db.transaction(PHOTOS, 'readwrite'); - const store = tx.objectStore(PHOTOS); - for (const p of photosToUpdate) { - store.put(p); - } - tx.oncomplete = () => resolve(); - tx.onerror = () => reject(tx.error); - }); - } - - return { reconnectedFolders, reconnectedPhotos }; } // The picker's own dialog; `id` makes the browser reopen at the folder this page @@ -1350,13 +1338,7 @@ export async function scanFolder( const file = await withTimeout(handle.getFile(), READ_MS); const id = photoId(folder.name, rel); const seen = known.get(id); - if (seen && seen.size === file.size && seen.mtime === file.lastModified) { - if (!seen.handle) { - batch.push({ ...seen, handle }); - if (due()) await flush(); - } - return; - } + if (seen && seen.size === file.size && seen.mtime === file.lastModified) return; // Where the frame was taken, off the few hundred kilobytes of its head that // its own EXIF lives in — where the 32MB a camera's RAW used to be read for // bought a preview this reading no longer wants. Nothing here looks at a @@ -1380,7 +1362,6 @@ export async function scanFolder( folder: normPath(folder.name), dir: cut < 0 ? '' : normPath(relNorm.slice(0, cut)), name: handle.name, - handle, taken: taken ?? file.lastModified, size: file.size, mtime: file.lastModified, @@ -1948,6 +1929,7 @@ export function cachedPhotos(): LibraryPhoto[] { export function clearLibraryCache(): void { cached = null; + dirHandles.clear(); } let backupTileFetcher: ((id: string) => Promise) | null = null; @@ -2167,7 +2149,7 @@ async function makeOneThumb(photo: LibraryPhoto): Promise { // A server's frame has no file to open: its tile is the one Immich drew, asked // for here and kept, so the second look comes out of the store and the sync's // own download is only paid once. - if (!photo.handle && photo.remoteId) { + if (photo.remoteId) { try { const res = await fetch(remoteUrl('thumbnail', photo)); if (!res.ok) return null; @@ -2178,7 +2160,6 @@ async function makeOneThumb(photo: LibraryPhoto): Promise { return null; } } - if (!photo.handle) return null; try { const file = await readPhotoFile(photo); const name = file.name; @@ -2298,12 +2279,32 @@ export async function readPhotos(folder?: string): Promise(PHOTOS, 'readonly', (s) => - folder ? s.index('folder').getAll(normPath(folder)) : s.getAll() - ); - for (let i = 0; i < rows.length; i++) { - if (rows[i].thumb) delete (rows[i] as { thumb?: Blob | null }).thumb; + // a handle for every frame of it into memory at once. An id is `folder/…`, so + // one folder's rows are the run of keys between the folder and its own name: + // the rows the index handed back, asked of the store, a chunk at a time. + const root = folder === undefined ? null : normPath(folder); + const at = (last: string | null): IDBKeyRange => + root === null + ? IDBKeyRange.lowerBound(last ?? '', last !== null) + : IDBKeyRange.bound(last ?? `${root}/`, `${root}/\uffff`, last !== null, false); + const rows: LibraryPhoto[] = []; + let last: string | null = null; + for (;;) { + const chunk = await new Promise((resolve, reject) => { + const req = db.transaction(PHOTOS, 'readonly').objectStore(PHOTOS).getAll(at(last), READ_CHUNK); + req.onsuccess = () => resolve(req.result as LibraryPhoto[]); + req.onerror = () => reject(req.error); + }); + if (!chunk.length) break; + last = chunk[chunk.length - 1].id; + for (const row of chunk) { + // Neither is read again: the file a frame is opened from is the folder's + // business — see `fileHandleFor` — and the tile is the store's own. + delete (row as { handle?: unknown }).handle; + delete (row as { thumb?: unknown }).thumb; + } + rows.push(...chunk); + if (chunk.length < READ_CHUNK) break; } rows.sort((a, b) => b.taken - a.taken); if (!folder) cached = rows; @@ -2381,8 +2382,9 @@ export async function readPhotoFile(photo: LibraryPhoto): Promise { const blob = await res.blob(); return new File([blob], photo.name, { type: blob.type || 'application/octet-stream' }); } - if (!photo.handle) throw new Error('no-file'); - return photo.handle.getFile(); + const handle = await fileHandleFor(photo); + if (!handle) throw new Error('no-file'); + return handle.getFile(); } // --- edits -----------------------------------------------------------------