From d0fddba1e8f153ae59a6549e9bb2fac1ea555755 Mon Sep 17 00:00:00 2001 From: 3dtours Date: Mon, 28 Sep 2026 19:55:02 +0700 Subject: [PATCH] web: walk a roll layer by layer, and let the strip drop the branch --- docker/frontend/scripts/library-check.mjs | 16 +++ docker/frontend/scripts/roll-walk-check.mjs | 99 +++++++++++++++++ docker/frontend/src/Library.tsx | 42 +++++++- docker/frontend/src/engine/library.ts | 113 ++++++++++---------- docker/frontend/src/engine/rollWalk.ts | 68 ++++++++++++ docker/frontend/src/i18n/en.ts | 3 + docker/frontend/src/i18n/vi.ts | 4 + 7 files changed, 282 insertions(+), 63 deletions(-) create mode 100644 docker/frontend/scripts/roll-walk-check.mjs create mode 100644 docker/frontend/src/engine/rollWalk.ts diff --git a/docker/frontend/scripts/library-check.mjs b/docker/frontend/scripts/library-check.mjs index 9460b71..fde4d9a 100644 --- a/docker/frontend/scripts/library-check.mjs +++ b/docker/frontend/scripts/library-check.mjs @@ -251,6 +251,22 @@ await page.waitForFunction(() => document.querySelectorAll('[data-key^="lib-node }); step('the same click draws it open again', true, '3 rows'); +// Show subfolders or not: the strip follows the open folder alone, or the whole +// branch under it. The switch is the reader's and the tree does not change with +// it — folding is about rows, this is about frames. +await page.click('[data-key="lib-subs"]'); +await page.waitForFunction((sel) => document.querySelectorAll(sel).length === 1, thumbs, { timeout: 10_000, polling: 50 }); +const shallow = await page.$$eval('[data-key^="lib-node-"]', (els) => els.length); +const shallowStage = await page.locator('[data-key="lib-stage-meta"]').innerText(); +step( + 'the switch leaves the open folder its own frames', + shallow === 3 && /P1010256\.JPG/.test(shallowStage), + `${shallow} rows, stage ${shallowStage.split('\n')[0]}` +); +await page.click('[data-key="lib-subs"]'); +await page.waitForFunction((sel) => document.querySelectorAll(sel).length === 3, thumbs, { timeout: 10_000, polling: 50 }); +step('and puts the branch back', true, '3 tiles from the roll below'); + // Opening the subfolder redraws the strip and the frame above it. await page.click('[data-key="lib-node-CheckRoll/2026"]'); await page.waitForFunction((sel) => document.querySelectorAll(sel).length === 2, thumbs, { timeout: 30_000 }); diff --git a/docker/frontend/scripts/roll-walk-check.mjs b/docker/frontend/scripts/roll-walk-check.mjs new file mode 100644 index 0000000..0f52979 --- /dev/null +++ b/docker/frontend/scripts/roll-walk-check.mjs @@ -0,0 +1,99 @@ +// The walk that reads a picked folder (src/engine/rollWalk.ts) is the part of a +// scan that can run for minutes, and its order is what the reader sees: parents +// before children, shallower layers before deeper ones, and the folder they just +// clicked in front of everything. None of that can be watched in a browser, so +// it is walked here over a stand-in tree. +// +// node scripts/roll-walk-check.mjs +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import ts from 'typescript'; + +const src = readFileSync(new URL('../src/engine/rollWalk.ts', import.meta.url), 'utf8'); +const js = ts.transpileModule(src, { + compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2022 }, +}).outputText; +const { DIRS_PER_PASS, jumpQueue, walkPass } = await import(`data:text/javascript,${encodeURIComponent(js)}`); + +// A directory handle is only ever read for `values()`, so a stand-in tree needs +// nothing else: `{ name: { … } }` for folders and `name: null` for a file. +// `@eaDir` and `.thumbnails` are the folders a real roll drags along. +const dir = (name, children) => ({ + kind: 'directory', + name, + values: async function* () { + for (const [child, kids] of Object.entries(children)) { + yield kids === null ? { kind: 'file', name: child } : dir(child, kids); + } + }, +}); +const keep = (name) => /\.(jpe?g|rw2)$/i.test(name); + +const roll = dir('CheckRoll', { + '.thumbnails': { hidden: null }, + '@eaDir': { junk: null }, + 'P1.JPG': null, + '2026': { 'P2.JPG': null, '04': { 'P3.RW2': null, 'notes.txt': null } }, + '2025': { 'P4.JPG': null }, +}); + +// Walks to the end, one click per pass, which is what the browser does between +// passes: whatever the reader clicked last is what `jump` names. +const walk = async (jumps = []) => { + const pending = [{ dir: roll, rel: '' }]; + const out = []; + let pass = 0; + let walked = false; + while (!walked) walked = await walkPass(pending, out, keep, () => jumps[pass++] ?? null); + return { out, passes: pass }; +}; + +// Nothing clicked: the picked folder's own frames, then a layer down, then the +// deepest — a folder is never walked before the folder that holds it. +const { out, passes } = await walk(); +const rels = out.map((f) => f.rel); +assert.deepEqual(rels, ['P1.JPG', '2025/P4.JPG', '2026/P2.JPG', '2026/04/P3.RW2'], 'layer by layer from the parent down'); +assert.ok( + rels.every((rel, i) => i === 0 || rel.split('/').length >= rels[i - 1].split('/').length), + 'no deeper frame is read before a shallower one' +); +assert.ok(!rels.some((rel) => /\.thumbnails|@eaDir|notes\.txt/.test(rel)), 'hidden and unsupported names are skipped'); +assert.equal(passes, 4, 'one folder a pass — the thread goes back to the page between them'); + +// A click on one of the two folders a layer down while its layer is still to be +// walked puts that folder first: the frames the reader is looking at are read +// before the branch beside them. +const clicked = await walk([null, '2026/']); +assert.deepEqual( + clicked.out.map((f) => f.rel), + ['P1.JPG', '2026/P2.JPG', '2025/P4.JPG', '2026/04/P3.RW2'], + 'the clicked folder jumps the queue' +); + +// And the same rule without a browser around it: a pass reads the queue in its +// own order unless the click says otherwise, and a name the queue does not hold +// changes nothing. +const wide = dir('Wide', { a: { 'A.JPG': null }, b: { 'B.JPG': null } }); +const before = []; +await walkPass([{ dir: wide, rel: '' }], before, keep, () => null); +assert.deepEqual(before.map((f) => f.rel), ['a/A.JPG', 'b/B.JPG'], 'untouched, the folders are read in the order they were found'); +const after = []; +await walkPass([{ dir: wide, rel: '' }], after, keep, () => 'b/'); +assert.deepEqual( + after.map((f) => f.rel), + ['b/B.JPG', 'a/A.JPG'], + 'the folder the reader clicked is read first' +); +assert.equal(jumpQueue([{ rel: 'a/' }], 'nowhere/'), undefined, 'a folder already walked, or never seen, is left alone'); + +// A pass is bounded: a library wider than the budget is read over several passes, +// and the rest waits in the queue rather than in a loop that never yields. +const children = {}; +for (let i = 0; i < DIRS_PER_PASS + 5; i++) children[`d${String(i).padStart(3, '0')}`] = { 'x.jpg': null }; +const many = [{ dir: dir('Many', children), rel: '' }]; +const found = []; +assert.equal(await walkPass(many, found, keep, () => null), false, 'the queue is not empty after one pass'); +assert.equal(found.length, DIRS_PER_PASS - 1, `one pass walked the picked folder and ${DIRS_PER_PASS - 1} below it`); +assert.equal(many.length, 6, 'the rest is left in the queue, for the next pass'); + +console.log(`ok roll walk — ${rels.length} frames layer by layer, a click jumps the queue, ${DIRS_PER_PASS} folders a pass`); diff --git a/docker/frontend/src/Library.tsx b/docker/frontend/src/Library.tsx index cbe6149..bff09f1 100644 --- a/docker/frontend/src/Library.tsx +++ b/docker/frontend/src/Library.tsx @@ -75,6 +75,10 @@ export function Library() { // Rows whose children are folded away. A collapse is a way of looking at the // tree, not a property of the roll, so it is not remembered between visits. const [folded, setFolded] = useState>(() => new Set()); + // Whether the strip draws the whole branch under the open node or only the + // node itself. Deep is how a roll reads — its top level holds no file of its + // own — and shallow is what a folder of folders is for. + const [deep, setDeep] = useState(true); const [up, setUp] = useState(null); const [view, setView] = useState<'preview' | 'grid'>('preview'); const [colWidth, setColWidth] = useState(() => { @@ -89,6 +93,10 @@ export function Library() { // one more folder. const [menu, setMenu] = useState<{ folder: LibraryFolder | null; x: number; y: number } | null>(null); const stopRef = useRef(false); + // The folder the reader last clicked, as the walk spells it ('' for the picked + // folder, `2026/04/` under it): the scan in flight pulls it to the front of + // its queue, so the branch in front of the eye is read first. + const jumpRef = useRef(null); // Where the drag of the column divider started, and how wide the column was // then — the pointer only ever reports a position, not a distance moved. const grip = useRef<{ x: number; w: number } | null>(null); @@ -166,7 +174,7 @@ export function Library() { setBusy(true); setNote(null); try { - const result = await scanFolder(folder, setProgress, () => stopRef.current); + const result = await scanFolder(folder, setProgress, () => stopRef.current, () => jumpRef.current); setNote(t('lib.scanned', { folder: folder.name, added: result.added, total: result.total })); await reload(); } catch { @@ -251,6 +259,12 @@ export function Library() { const rootName = node ? node.split('/')[0] : null; + // A row's folder as the walk spells it: '' for the picked folder itself, the + // path under it for a subfolder, and nothing for a row belonging to another + // folder — that one is not a folder of this roll. + const relOf = (key: string): string | null => + !rootName ? null : key === rootName ? '' : key.startsWith(`${rootName}/`) ? `${key.slice(rootName.length + 1)}/` : null; + // The tree: one row per picked folder, then one per subfolder a frame sits in, // indented by its depth, each carrying how many frames it holds at or under it. const nodes = useMemo(() => { @@ -316,13 +330,15 @@ export function Library() { setNode(nodes.find((n) => n.key === node.split('/')[0])?.key ?? nodes[0]?.key ?? null); }, [loaded, node, nodes]); - // What the open node draws: its own frames, and the ones in every subfolder - // under it — a roll's top level usually holds no file of its own. + // What the open node draws: its own frames, and — unless the branch has been + // shut out of the strip — the ones in every subfolder under it, because a + // roll's top level usually holds no file of its own. const shown = useMemo(() => { if (!rootName) return photos; const dir = node === rootName ? '' : (node ?? '').slice(rootName.length + 1); - return photos.filter((p) => p.folder === rootName && (dir === '' || p.dir === dir || p.dir.startsWith(`${dir}/`))); - }, [node, photos, rootName]); + const below = dir === '' ? (p: LibraryPhoto) => p.dir !== '' : (p: LibraryPhoto) => p.dir.startsWith(`${dir}/`); + return photos.filter((p) => p.folder === rootName && (p.dir === dir || (deep && below(p)))); + }, [deep, node, photos, rootName]); const frame = shown.find((p) => p.id === up) ?? shown[0] ?? null; const urls = useThumbUrls(shown); @@ -415,6 +431,19 @@ export function Library() { + {/* Whether the strip draws the branch under the open folder or + only the folder itself. The label says what the click does, + `on` says where the strip is now. */} + ) : null} @@ -465,6 +494,9 @@ export function Library() { // the rows go. onClick={() => { setNode(n.key); + // The folder just clicked is the one a scan in flight + // reads next, whatever layer it sits in. + jumpRef.current = relOf(n.key); if (!under) return; setFolded((prev) => { const next = new Set(prev); diff --git a/docker/frontend/src/engine/library.ts b/docker/frontend/src/engine/library.ts index 4412d89..1fffbf3 100644 --- a/docker/frontend/src/engine/library.ts +++ b/docker/frontend/src/engine/library.ts @@ -15,6 +15,7 @@ import type { Recipe } from '../../shared/types'; import { readCapturedAt } from './imageOps'; import { isRawName, rawThumbnail } from './rawDevelop'; +import { walkPass, type WalkFile } from './rollWalk'; const DB_NAME = 'recipescam-library'; const FOLDERS = 'folders'; @@ -235,32 +236,13 @@ export interface ScanProgress { added: number; } -// The walk goes all the way down: a picked folder is usually a roll of -// date-named subfolders, and a catalogue that only saw its top level would be -// empty. Names beginning with `.` or `@` are skipped — that is `.thumbnails`, -// `.git`, and the `@eaDir` a Synology writes beside every frame. -async function collectFiles( - dir: FileSystemDirectoryHandle, - prefix: string, - out: { handle: FileSystemFileHandle; rel: string }[] -): Promise { - for await (const entry of dir.values()) { - if (/^[.@]/.test(entry.name)) continue; - if (entry.kind === 'directory') { - try { - await collectFiles(entry as FileSystemDirectoryHandle, `${prefix}${entry.name}/`, out); - } catch { - // A subfolder that will not open is skipped; the rest of the roll stands. - } - } else if (isSupportedPhoto(entry.name)) { - out.push({ handle: entry as FileSystemFileHandle, rel: `${prefix}${entry.name}` }); - } - } -} - // Walk the folder, keep what is new or changed, and leave the rest alone: a // second scan of a 2000-file folder only reads the frames that moved. // +// The walk itself lives in `rollWalk`: layer by layer from the picked folder +// down, one bounded pass at a time, with the folder the reader just clicked +// pulled to the front of the queue. `jump` is that click. +// // ponytail: read one frame at a time, whole, on the main thread — a RAW is read // as its bytes for LibRaw and released again. A folder of 5000 RAW files takes // minutes and makes the page lumpy while it runs. Move the walk into a worker @@ -268,12 +250,13 @@ async function collectFiles( export async function scanFolder( folder: LibraryFolder, onProgress?: (p: ScanProgress) => void, - shouldStop?: () => boolean + shouldStop?: () => boolean, + jump?: () => string | null ): Promise { const known = new Map((await listPhotos(folder.name)).map((p) => [p.id, p])); - const entries: { handle: FileSystemFileHandle; rel: string }[] = []; - await collectFiles(folder.handle, '', entries); - const progress: ScanProgress = { folder: folder.name, total: entries.length, done: 0, added: 0 }; + const pending = [{ dir: folder.handle, rel: '' }]; + const entries: WalkFile[] = []; + const progress: ScanProgress = { folder: folder.name, total: 0, done: 0, added: 0 }; let batch: LibraryPhoto[] = []; // One transaction per batch, a put per row: a store with `keyPath: 'id'` takes // a record, not an array of them. @@ -290,39 +273,53 @@ export async function scanFolder( tx.onerror = () => reject(tx.error); }); }; - for (const { handle, rel } of entries) { - if (shouldStop?.()) break; - progress.done++; - try { - const file = await handle.getFile(); - const id = photoId(folder.name, rel); - const seen = known.get(id); - if (seen && seen.size === file.size && seen.taken && seen.taken === file.lastModified && seen.thumb) { - onProgress?.(progress); - continue; + let stop = false; + let walked = false; + // One bounded pass at a time, and the pass yields between frames, so a folder + // clicked while the scan runs is walked on the next jump rather than after the + // whole tree. Nothing clicked: parents first, then each layer below, deepest + // last — the order `pending` is filled in. + do { + const from = entries.length; + walked = await walkPass(pending, entries, isSupportedPhoto, jump ?? (() => null)); + progress.total += entries.length - from; + for (const { handle, rel } of entries.slice(from)) { + if (shouldStop?.()) { + stop = true; + break; } - const bytes = new Uint8Array(await file.arrayBuffer()); - const [thumb, taken] = [await makeThumbnail(file, bytes), (await readCapturedAt(bytes)) ?? file.lastModified]; - const cut = rel.lastIndexOf('/'); - batch.push({ - id, - folder: folder.name, - dir: cut < 0 ? '' : rel.slice(0, cut), - name: handle.name, - handle, - thumb, - taken, - size: file.size, - addedAt: seen?.addedAt ?? Date.now(), - }); - progress.added++; - if (batch.length >= BATCH) await flush(); - } catch { - // A frame that will not read is a frame the catalogue skips: one bad file - // in a folder is not a failed folder. + progress.done++; + try { + const file = await handle.getFile(); + const id = photoId(folder.name, rel); + const seen = known.get(id); + if (seen && seen.size === file.size && seen.taken && seen.taken === file.lastModified && seen.thumb) { + onProgress?.(progress); + continue; + } + const bytes = new Uint8Array(await file.arrayBuffer()); + const [thumb, taken] = [await makeThumbnail(file, bytes), (await readCapturedAt(bytes)) ?? file.lastModified]; + const cut = rel.lastIndexOf('/'); + batch.push({ + id, + folder: folder.name, + dir: cut < 0 ? '' : rel.slice(0, cut), + name: handle.name, + handle, + thumb, + taken, + size: file.size, + addedAt: seen?.addedAt ?? Date.now(), + }); + progress.added++; + if (batch.length >= BATCH) await flush(); + } catch { + // A frame that will not read is a frame the catalogue skips: one bad file + // in a folder is not a failed folder. + } + onProgress?.(progress); } - onProgress?.(progress); - } + } while (!walked && !stop); await flush(); return progress; } diff --git a/docker/frontend/src/engine/rollWalk.ts b/docker/frontend/src/engine/rollWalk.ts new file mode 100644 index 0000000..bb2d76c --- /dev/null +++ b/docker/frontend/src/engine/rollWalk.ts @@ -0,0 +1,68 @@ +// Walking a picked folder is the one part of the catalogue that can run for +// minutes: a roll is dated folders inside dated folders, and a library can hold +// thousands. The walk goes from the parent down, one layer at a time, so the +// frames nearest the top are indexed first — and a folder the reader clicks is +// put in front of the queue, so the branch in front of them is read before the +// layer it sits in would have come round. A pass covers `DIRS_PER_PASS` +// directories and then hands the thread back, which is what keeps the page +// answering while a scan runs. +// +// Pure on purpose: it only reads `values()` off a directory handle, so a check +// can walk a stand-in tree without a browser (scripts/roll-walk-check.mjs). + +export interface WalkFile { + handle: FileSystemFileHandle; + rel: string; +} + +interface Pending { + dir: FileSystemDirectoryHandle; + rel: string; +} + +// Directories one pass walks before it yields: small enough that a click is +// answered within a second, large enough that a folder of a few hundred is one +// pass rather than a hundred. +export const DIRS_PER_PASS = 200; + +// The folder the reader clicked goes first. `rel` is how the queue spells a +// folder — '' for the picked one, `2026/04/` under it — and a folder already +// walked is simply not in the queue, which is the no-op case. +// +// ponytail: only a folder the queue already holds can jump, which is every +// folder the tree can draw, because a row exists per indexed frame. A tree that +// listed folders before their first frame would need `getDirectoryHandle` to +// descend to the clicked path here. +export function jumpQueue(pending: Pending[], rel: string | null): void { + if (rel === null) return; + const at = pending.findIndex((p) => p.rel === rel); + if (at > 0) pending.unshift(pending.splice(at, 1)[0]); +} + +// One pass over the queue. The queue is the walk's own state: what is behind has +// been walked, what is ahead is in the order it was found — parents before +// children, shallower layers before deeper ones. Returns true when nothing is +// left to walk. +export async function walkPass( + pending: Pending[], + out: WalkFile[], + keep: (name: string) => boolean, + jump: () => string | null +): Promise { + for (let i = 0; i < DIRS_PER_PASS && pending.length; i++) { + jumpQueue(pending, jump()); + const here = pending.shift()!; + try { + for await (const entry of here.dir.values()) { + // Names beginning with `.` or `@` are skipped — that is `.thumbnails`, + // `.git`, and the `@eaDir` a Synology writes beside every frame. + if (/^[.@]/.test(entry.name)) continue; + if (entry.kind === 'directory') pending.push({ dir: entry as FileSystemDirectoryHandle, rel: `${here.rel}${entry.name}/` }); + else if (keep(entry.name)) out.push({ handle: entry as FileSystemFileHandle, rel: `${here.rel}${entry.name}` }); + } + } catch { + // A subfolder that will not open is skipped; the rest of the roll stands. + } + } + return pending.length === 0; +} diff --git a/docker/frontend/src/i18n/en.ts b/docker/frontend/src/i18n/en.ts index af0828a..db7df65 100644 --- a/docker/frontend/src/i18n/en.ts +++ b/docker/frontend/src/i18n/en.ts @@ -240,6 +240,9 @@ export const en: Dict = { 'lib.colWidth': 'Drag to set the width of the folder column (arrow keys when focused)', 'lib.viewPreview': 'SINGLE', 'lib.viewGrid': 'ALL THUMBS', + 'lib.subsShow': 'SHOW SUBFOLDERS', + 'lib.subsHide': 'THIS FOLDER ONLY', + 'lib.subsHint': 'Frames in subfolders: the whole branch under the open folder, or only what that folder holds itself.', 'lib.open': 'Open in the studio', 'lib.edited': 'edited', 'lib.size': '{mb} MB', diff --git a/docker/frontend/src/i18n/vi.ts b/docker/frontend/src/i18n/vi.ts index cfac199..4874053 100644 --- a/docker/frontend/src/i18n/vi.ts +++ b/docker/frontend/src/i18n/vi.ts @@ -253,6 +253,10 @@ export const vi = { 'lib.colWidth': 'Kéo để chỉnh độ rộng cột thư mục (mũi tên trái/phải khi đang chọn)', 'lib.viewPreview': 'MỘT ẢNH', 'lib.viewGrid': 'TOÀN BỘ', + 'lib.subsShow': 'HIỆN CẢ THƯ MỤC CON', + 'lib.subsHide': 'CHỈ THƯ MỤC ĐANG CHỌN', + 'lib.subsHint': + 'Ảnh trong thư mục con: hiện cả nhánh dưới thư mục đang chọn, hoặc chỉ ảnh của chính thư mục đó.', 'lib.open': 'Mở trong studio', 'lib.edited': 'đã chỉnh', 'lib.size': '{mb} MB',