diff --git a/docker/frontend/scripts/library-check.mjs b/docker/frontend/scripts/library-check.mjs index 0cec1b9..88669b7 100644 --- a/docker/frontend/scripts/library-check.mjs +++ b/docker/frontend/scripts/library-check.mjs @@ -133,14 +133,22 @@ await context.addInitScript( const fileHandle = (name) => { const handle = { kind: 'file', name, __fake: true }; Object.defineProperty(handle, 'getFile', { - value: async () => - new File([await load(name)], name, { + value: async () => { + const gets = JSON.parse(sessionStorage.getItem('check.gets') ?? '{}'); + gets[name] = (gets[name] ?? 0) + 1; + sessionStorage.setItem('check.gets', JSON.stringify(gets)); + // A reading caught in the middle of a frame: the check sets the key, the + // scan stops on the frame it was reading, and the next visit — with the + // key cleared — is the one that goes on. + while (localStorage.getItem('check.hold') === '1') await new Promise((r) => setTimeout(r, 25)); + return new File([await load(name)], name, { type: name.endsWith('.JPG') ? 'image/jpeg' : 'application/octet-stream', // A real handle reports one write time for one file, and the same one // on every read; a fresh `Date.now()` per call would make every frame // look touched and hide what is being checked. lastModified: Date.UTC(2026, 0, 1), - }), + }); + }, }); return handle; }; @@ -154,14 +162,37 @@ await context.addInitScript( const handle = { kind: 'directory', name: label, __fake: true }; Object.defineProperties(handle, { values: { - value: () => - [ + value: () => { + // Every listing a reading asks for, counted across reloads: a reading + // that starts over lists the same folder twice, and that is the one + // thing on screen a reader reads as "it threw the catalogue away". + const lists = JSON.parse(sessionStorage.getItem('check.lists') ?? '{}'); + lists[label] = (lists[label] ?? 0) + 1; + sessionStorage.setItem('check.lists', JSON.stringify(lists)); + return [ ...(node.files ?? []).map(fileHandle), ...Object.entries(node.dirs ?? {}).map(([name, kid]) => dirHandle(name, kid)), - ][Symbol.iterator](), + ][Symbol.iterator](); + }, }, queryPermission: { value: async () => (window.__deny ? 'denied' : 'granted') }, requestPermission: { value: async () => (window.__deny ? 'denied' : 'granted') }, + // A folder is asked for a child by name as often as it is listed: the + // reading spells its queue as paths, and a path comes back as a handle + // this way. A name that is not there is the error the browser gives. + getDirectoryHandle: { + value: async (name) => { + const kid = node.dirs?.[name]; + if (!kid) throw new Error(`no folder named ${name}`); + return dirHandle(name, kid); + }, + }, + getFileHandle: { + value: async (name) => { + if (!(node.files ?? []).includes(name)) throw new Error(`no frame named ${name}`); + return fileHandle(name); + }, + }, }); return handle; }; @@ -678,6 +709,73 @@ await page .catch(() => {}); step('and taking it reads the folder again', (await page.$$eval(thumbs, (els) => els.length)) === 3); +// A reload in the middle of a reading — the reader holding Ctrl+Shift+R over a +// roll of a few thousand frames, which is the one thing they do while a reading +// runs. The catalogue holds the frames the reading got through, but the reading's +// own position is not in the catalogue: without it the walk starts at the top of +// the roll again, every folder is listed a second time, and the counter starts +// from zero while the frames it already holds are read again to be skipped. +// +// The position belongs to the tab now, so the reading carries on instead of +// starting over. `check.hold` stops the reading on the frame it is reading, which +// is where a reload lands — and the reading it left behind is still there when the +// key goes, so the same reading finishes the roll. Everything is taken out of the +// catalogue first, so the reading has frames on both sides of the reload. +await page.evaluate(async () => { + sessionStorage.clear(); + localStorage.setItem('check.hold', '1'); + const db = await new Promise((resolve, reject) => { + const req = indexedDB.open('recipescam-library'); + req.onsuccess = () => resolve(req.result); + req.onerror = () => reject(req.error); + }); + await new Promise((resolve, reject) => { + const tx = db.transaction('photos', 'readwrite'); + tx.objectStore('photos').clear(); + tx.oncomplete = () => resolve(); + tx.onerror = () => reject(tx.error); + }); +}); +// The position the tab is holding, the line the toolbar reads out, and the wait +// for it: the line is how far the reading has come, and it is the one thing on +// screen that says whether a reading started over. +const position = () => page.evaluate(() => JSON.parse(sessionStorage.getItem('recipescam.library.walk.CheckRoll') ?? 'null')); +const readout = () => page.$eval('[data-key="lib-progress"]', (e) => e.textContent ?? '').catch(() => ''); +const counting = () => + page.waitForFunction( + () => /\d+\/\d+/.test(document.querySelector('[data-key="lib-progress"]')?.textContent ?? ''), + null, + { timeout: 30_000, polling: 50 } + ); +await page.reload(); +await counting().catch(() => {}); +const stopped = await readout(); +await page.reload(); +await counting().catch(() => {}); +const resumed = await readout(); +const at = await position(); +step( + 'a reading a reload cut in half goes on from where it was', + /\d+\/\d+/.test(stopped) && resumed === stopped && at?.walked.length === 4 && at.pending.length === 0 && at.frames.length === 3, + `stopped at "${stopped.trim()}", back at "${resumed.trim()}", position ${JSON.stringify(at && { walked: at.walked.length, pending: at.pending.length, frames: at.frames.length })}` +); + +// The key goes, and the reading the reload interrupted finishes the roll on its +// own: the frames it had found are read, the folders are not listed again — the +// check counts every listing a reading asks for, across reloads — and the position +// is dropped once there is nothing left to carry on from. +await page.evaluate(() => localStorage.removeItem('check.hold')); +await page + .waitForFunction(() => document.querySelectorAll('[data-key^="lib-thumb-"]').length === 3, null, { timeout: 60_000, polling: 50 }) + .catch(() => {}); +const carried = await page.$$eval(thumbs, (els) => els.length).catch(() => 0); +const again = await page.evaluate(() => JSON.parse(sessionStorage.getItem('check.lists') ?? '{}')); +step( + 'and it finishes the roll without listing a folder twice', + carried === 3 && again.CheckRoll === 1 && again['2026'] === 1 && again.Empty === 1 && again['04'] === 1 && (await position()) === null, + `${carried} tiles, folders listed ${JSON.stringify(again)}` +); + await browser.close(); sampleServer.close(); app.kill('SIGTERM'); diff --git a/docker/frontend/src/engine/library.ts b/docker/frontend/src/engine/library.ts index 3e44c6f..7a058bd 100644 --- a/docker/frontend/src/engine/library.ts +++ b/docker/frontend/src/engine/library.ts @@ -15,7 +15,7 @@ import type { Recipe } from '../../shared/types'; import { readCapturedAt } from './imageOps'; import { isRawName, rawThumbnail } from './rawDevelop'; -import { walkPass, type Walk, type WalkFile } from './rollWalk'; +import { openAt, walkPass, type Walk, type WalkFile } from './rollWalk'; const DB_NAME = 'recipescam-library'; const FOLDERS = 'folders'; @@ -187,6 +187,9 @@ export async function removeFolder(name: string): Promise { tx.oncomplete = () => resolve(); tx.onerror = () => reject(tx.error); }); + // The reading's position goes with the folder: a folder that is gone is not + // one to carry on reading, and a folder picked again starts at the top. + clearWalk(name); } // A handle kept in IndexedDB comes back without its permission: the browser @@ -323,6 +326,105 @@ export interface ScanProgress { // elsewhere (disk, the decoder's own threads), so what is left here is small — // but a 5000-frame folder still makes the page lumpy. Move the walk into a // worker (LibRaw is happy in one) if a catalogue that large is ever real. + +// Where the reading had got to. A reload in the middle of a few thousand frames +// takes the page with it, and what it would take from the reading is the +// position: the folders the walk had been through, the ones still in its queue, +// and the frames it had found and not read. None of that is in the catalogue — +// the catalogue only holds the frames that were read — so the reading would +// start again at the top of the roll and walk every folder a second time. +// +// Session storage holds exactly that position, and holds it because it is the +// tab's: the tab that reloads carries on from the frame it stopped at, and a new +// tab starts a reading of its own from the top. The key goes when the reading +// finishes. +const WALK_KEY = (folder: string) => `recipescam.library.walk.${folder}`; + +interface WalkSaved { + // The folders the walk is through, and the ones it has not read yet, in the + // order it was going to read them. + walked: string[]; + pending: string[]; + // The frames the walk has found and the reading has not got to. The front of + // this is the frame a reloaded reading picks up at. + frames: string[]; + // How many of the front of `frames` were already counted as read by the reading + // that wrote this down: the rows are in hand and not in the catalogue yet, so + // the next reading reads them again — and counts none of them twice. + restored: number; + progress: ScanProgress; +} + +function loadWalk(folder: string): WalkSaved | null { + try { + const raw = sessionStorage.getItem(WALK_KEY(folder)); + if (!raw) return null; + const saved = JSON.parse(raw) as WalkSaved; + return Array.isArray(saved?.walked) && + Array.isArray(saved?.pending) && + Array.isArray(saved?.frames) && + Number.isInteger(saved?.restored) && + saved.progress + ? saved + : null; + } catch { + return null; + } +} + +// A position that cannot be written down is a reading that starts from the top, +// which is what this did before — private mode and a full quota both land here. +function saveWalk( + folder: string, + walked: string[], + pending: string[], + frames: string[], + restored: number, + progress: ScanProgress +): void { + try { + sessionStorage.setItem(WALK_KEY(folder), JSON.stringify({ walked, pending, frames, restored, progress } satisfies WalkSaved)); + } catch { + // Nothing to write down: the reading starts from the top, as it did before. + } +} + +function clearWalk(folder: string): void { + try { + sessionStorage.removeItem(WALK_KEY(folder)); + } catch { + // Nothing to clear. + } +} + +// The frames a written-down position names, asked of the root again: a frame is +// spelled by its path, and a handle is not a thing that can be written down. The +// folder part is asked once and kept, because a reading that stopped in a folder +// of a few thousand frames has nearly all of them in the same folder. +async function framesAt(root: FileSystemDirectoryHandle, rels: string[]): Promise { + const dirs = new Map(); + const out: WalkFile[] = []; + for (const rel of rels) { + const cut = rel.lastIndexOf('/'); + const at = cut < 0 ? '' : rel.slice(0, cut); + let dir = dirs.get(at); + if (!dir) { + const opened = await openAt(root, at); + // A folder that will not open takes its frames with it; they are read + // again when the walk reaches it, if it opens at all. + if (!opened) continue; + dirs.set(at, opened); + dir = opened; + } + try { + out.push({ handle: await dir.getFileHandle(rel.slice(cut + 1)), rel }); + } catch { + // A frame that is gone is not a frame to read. + } + } + return out; +} + export async function scanFolder( folder: LibraryFolder, onProgress?: (p: ScanProgress) => void, @@ -330,15 +432,37 @@ export async function scanFolder( jump?: () => string | null ): Promise { const known = new Map((await listPhotos(folder.name)).map((p) => [p.id, p])); - const tree: Walk = { root: folder.handle, pending: [{ dir: folder.handle, rel: '' }], walked: new Set() }; - const entries: WalkFile[] = []; + const saved = loadWalk(folder.name); + const tree: Walk = { root: folder.handle, pending: [], walked: new Set(saved?.walked ?? []) }; + // A queue comes back as paths, so the folders are asked for again; a reading + // with nothing written down starts at the picked folder. + if (saved) { + for (const rel of saved.pending) { + const dir = await openAt(folder.handle, rel); + if (dir) tree.pending.push({ dir, rel }); + } + } else { + tree.pending.push({ dir: folder.handle, rel: '' }); + } + // The frames this reading has found and not read: the walk fills it, the lanes + // empty it, and a frame taken off the front is one a reload will not read + // again. Also where a reading that was cut off picks up. + const entries: WalkFile[] = saved ? await framesAt(folder.handle, saved.frames) : []; // What the passes found: the folders this scan walked into, and the names they // put in the column. `names` is cleared a pass at a time; `found` keeps the // whole scan so a folder that has gone from the disk goes from the column too. const names: string[] = []; - const found = new Set(); - const progress: ScanProgress = { folder: folder.name, total: 0, done: 0, added: 0, dirs: [], counts: {} }; - const counts: Record = {}; + const progress: ScanProgress = saved?.progress ?? { + folder: folder.name, + total: 0, + done: 0, + added: 0, + dirs: [], + counts: {}, + }; + progress.folder = folder.name; + const found = new Set(progress.dirs.map((d) => d.id)); + const counts: Record = progress.counts; // Count the frame the moment the scan gets to it, before it knows whether the // frame is new: the rows say how far the reading has come, not what it kept. const count = (rel: string) => { @@ -354,11 +478,33 @@ export async function scanFolder( progress.counts = { ...counts }; }; // The folders this scan has walked into, kept for the whole of it: the column - // is redrawn from this list, so it only ever grows. - const dirs: LibraryDir[] = []; + // is redrawn from this list, so it only ever grows. A reading that was cut off + // hands the list on with the rest of its position. + const dirs: LibraryDir[] = progress.dirs; let batch: LibraryPhoto[] = []; + // The frames in hand: the ones a lane has been handed, and the ones whose row is + // in `batch` and not yet in the catalogue. They are on neither side of the line a + // position is written on — the catalogue does not hold them and the walk will not + // find them again — so a reload has to read them again, and they are also the + // frames the written-down count has already counted. + let inHand: string[] = []; + const pathOf = (row: LibraryPhoto) => (row.dir ? `${row.dir}/${row.name}` : row.name); + const write = () => { + const held = [...inHand, ...batch.map(pathOf)]; + saveWalk( + folder.name, + [...tree.walked], + tree.pending.map((p) => p.rel), + [...held, ...entries.map((e) => e.rel)], + held.length, + progress + ); + }; // One transaction per batch, a put per row: a store with `keyPath: 'id'` takes - // a record, not an array of them. + // a record, not an array of them. The position is written with it — a batch is + // the beat the reading and the stored position are kept in step at, so at most + // one batch is read again after a reload, and a frame that is read again is + // skipped on its size and its time. const flush = async () => { if (!batch.length) return; const rows = batch; @@ -371,6 +517,7 @@ export async function scanFolder( tx.oncomplete = () => resolve(); tx.onerror = () => reject(tx.error); }); + write(); }; // One frame, end to end: its bytes, its tile, its shutter time, and the row the // catalogue keeps. A frame that has not moved is dropped on its size and its @@ -409,46 +556,34 @@ export async function scanFolder( addedAt: seen?.addedAt ?? Date.now(), }); progress.added++; + inHand = inHand.filter((r) => r !== rel); if (batch.length >= BATCH) await flush(); }; - let stop = false; - let walked = false; - // One bounded pass at a time, and the pass yields between frames, so a folder - // clicked while the scan runs is read on the next jump rather than after the - // whole tree. Nothing clicked: the picked folder and its frames, then each - // layer below with theirs, deepest last — the order `pending` is filled in. - do { - const from = entries.length; - names.length = 0; - walked = await walkPass(tree, entries, names, isSupportedPhoto, jump ?? (() => null)); - // The names come in before the frames they hold: the column grows one pass - // ahead of the strip, which is the whole point of reading layer by layer. - for (const rel of names) { - const id = `${folder.name}/${rel.slice(0, -1)}`; - if (found.has(id)) continue; - found.add(id); - dirs.push({ id, folder: folder.name, rel: rel.slice(0, -1) }); - } - // The column is handed the names the moment the pass is through, before a - // single frame under them has been read. A copy, so the redraw has something - // new to look at rather than the list growing under it. - progress.dirs = dirs.slice(); - progress.total += entries.length - from; - // A few frames at a time, not one: reading a frame is almost all waiting — - // the bytes come off the disk, the decode runs on a thread of its own — and - // the wait of one frame is spent on the next one instead of sitting still. - // Lanes are why the same roll lands in a fraction of the time; past a few the - // disk and the decoder are the limit. The column still counts a frame the - // moment the scan reaches it, so which frames are up is unchanged. + // The frames in hand, read a few at a time: reading a frame is almost all + // waiting — the bytes come off the disk, the decode runs on a thread of its own + // — and the wait of one frame is spent on the next one instead of sitting + // still. Lanes are why the same roll lands in a fraction of the time; past a + // few the disk and the decoder are the limit. A frame leaves the queue the + // moment its lane is handed it, so the position on disk trails the reading by + // no more than one batch. + const drain = async () => { const lanes = new Set>(); - for (const { handle, rel } of entries.slice(from)) { + while (entries.length) { if (shouldStop?.()) { stop = true; break; } - progress.done++; - count(rel); + const { handle, rel } = entries.shift()!; + inHand.push(rel); + // A frame a reload handed back that was counted already is read without + // being counted twice; the ones behind it are frames that reading found and + // never reached, and they are counted here as they are reached. + if (restored) restored--; + else { + progress.done++; + count(rel); + } const lane = readOne(handle, rel) .catch(() => undefined) .finally(() => lanes.delete(lane)); @@ -457,8 +592,57 @@ export async function scanFolder( onProgress?.(progress); } await Promise.all(lanes); + }; + + // How many of the frames in hand were counted by the reading that handed them + // back: the rows it had read and not yet stored. The rest of the queue is frames + // it found and never got to, and this reading counts those as it reaches them — + // the rows and the counter both say how far the reading has come, and a reload + // that stopped at a frame must not count it twice. + let restored = Math.min(saved?.restored ?? 0, entries.length); + + let stop = false; + let walked = false; + // One bounded pass at a time, and the pass yields between frames, so a folder + // clicked while the scan runs is read on the next jump rather than after the + // whole tree. Nothing clicked: the picked folder and its frames, then each + // layer below with theirs, deepest last — the order `pending` is filled in. + do { + names.length = 0; + // The queue a reading that was cut off left behind goes first, and the walk + // is asked for nothing until it is through: the frames already found are + // what that reading was in the middle of. + if (!entries.length) { + walked = await walkPass(tree, entries, names, isSupportedPhoto, jump ?? (() => null)); + // The names come in before the frames they hold: the column grows one pass + // ahead of the strip, which is the whole point of reading layer by layer. + for (const rel of names) { + const id = `${folder.name}/${rel.slice(0, -1)}`; + if (found.has(id)) continue; + found.add(id); + dirs.push({ id, folder: folder.name, rel: rel.slice(0, -1) }); + } + // The column is handed the names the moment the pass is through, before a + // single frame under them has been read. A copy, so the redraw has + // something new to look at rather than the list growing under it, and a + // reading that is cut off hands the same list on with its position. + progress.dirs = dirs.slice(); + // Every frame found is a frame the reading has or will reach, so what is + // still in the queue is what is left of the total. + progress.total = progress.done + entries.length; + // The position before a frame of this pass is looked at: a reload here + // reads the pass's frames again, and a frame already in the catalogue is + // skipped anyway — where the other order would leave the last pass's + // frames out of the catalogue until the next visit. + write(); + onProgress?.(progress); + } + await drain(); } while (!walked && !stop); await flush(); + // A reading that came to its end has no position worth keeping: the next one + // walks the roll from the top and skips what has not moved. + if (!stop) clearWalk(folder.name); return progress; } diff --git a/docker/frontend/src/engine/rollWalk.ts b/docker/frontend/src/engine/rollWalk.ts index 48dd033..fdcf3af 100644 --- a/docker/frontend/src/engine/rollWalk.ts +++ b/docker/frontend/src/engine/rollWalk.ts @@ -46,6 +46,19 @@ export const DIRS_PER_PASS = 200; // `$Recycle.Bin` on the next. export const SYSTEM_DIR = /^[.@]|^\$RECYCLE\.BIN$|^RECYCLER$|^System Volume Information$|^#recycle$|^lost\+found$/i; +// The folder a path names, asked from the root one segment at a time. A queue +// that was written down as paths comes back as handles this way, and a path that +// will not open answers null rather than a folder that is not there. +export async function openAt(root: FileSystemDirectoryHandle, rel: string): Promise { + let dir = root; + try { + for (const part of rel.split('/').filter(Boolean)) dir = await dir.getDirectoryHandle(part); + } catch { + return null; + } + return dir; +} + // The folder the reader clicked goes first. When it is not in the queue the walk // has not reached it yet — its row is on screen because the parent pass named it // — so it is opened from the picked folder and read next. A folder already read, @@ -57,14 +70,8 @@ export async function jumpQueue(walk: Walk, rel: string | null): Promise { if (at > 0) walk.pending.unshift(walk.pending.splice(at, 1)[0]); return; } - let dir = walk.root; - for (const part of rel.split('/').filter(Boolean)) { - try { - dir = await dir.getDirectoryHandle(part); - } catch { - return; - } - } + const dir = await openAt(walk.root, rel); + if (!dir) return; // ponytail: the reader's folder jumps the whole queue, ancestors included — // the folders between it and the picked one are read when their layer comes. walk.pending.unshift({ dir, rel });