web: walk a roll layer by layer, and let the strip drop the branch

This commit is contained in:
2026-09-28 19:55:02 +07:00
parent bd206a8be9
commit d0fddba1e8
7 changed files with 282 additions and 63 deletions
+16
View File
@@ -251,6 +251,22 @@ await page.waitForFunction(() => document.querySelectorAll('[data-key^="lib-node
});
step('the same click draws it open again', true, '3 rows');
// Show subfolders or not: the strip follows the open folder alone, or the whole
// branch under it. The switch is the reader's and the tree does not change with
// it — folding is about rows, this is about frames.
await page.click('[data-key="lib-subs"]');
await page.waitForFunction((sel) => document.querySelectorAll(sel).length === 1, thumbs, { timeout: 10_000, polling: 50 });
const shallow = await page.$$eval('[data-key^="lib-node-"]', (els) => els.length);
const shallowStage = await page.locator('[data-key="lib-stage-meta"]').innerText();
step(
'the switch leaves the open folder its own frames',
shallow === 3 && /P1010256\.JPG/.test(shallowStage),
`${shallow} rows, stage ${shallowStage.split('\n')[0]}`
);
await page.click('[data-key="lib-subs"]');
await page.waitForFunction((sel) => document.querySelectorAll(sel).length === 3, thumbs, { timeout: 10_000, polling: 50 });
step('and puts the branch back', true, '3 tiles from the roll below');
// Opening the subfolder redraws the strip and the frame above it.
await page.click('[data-key="lib-node-CheckRoll/2026"]');
await page.waitForFunction((sel) => document.querySelectorAll(sel).length === 2, thumbs, { timeout: 30_000 });
@@ -0,0 +1,99 @@
// The walk that reads a picked folder (src/engine/rollWalk.ts) is the part of a
// scan that can run for minutes, and its order is what the reader sees: parents
// before children, shallower layers before deeper ones, and the folder they just
// clicked in front of everything. None of that can be watched in a browser, so
// it is walked here over a stand-in tree.
//
// node scripts/roll-walk-check.mjs
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import ts from 'typescript';
const src = readFileSync(new URL('../src/engine/rollWalk.ts', import.meta.url), 'utf8');
const js = ts.transpileModule(src, {
compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2022 },
}).outputText;
const { DIRS_PER_PASS, jumpQueue, walkPass } = await import(`data:text/javascript,${encodeURIComponent(js)}`);
// A directory handle is only ever read for `values()`, so a stand-in tree needs
// nothing else: `{ name: { … } }` for folders and `name: null` for a file.
// `@eaDir` and `.thumbnails` are the folders a real roll drags along.
const dir = (name, children) => ({
kind: 'directory',
name,
values: async function* () {
for (const [child, kids] of Object.entries(children)) {
yield kids === null ? { kind: 'file', name: child } : dir(child, kids);
}
},
});
const keep = (name) => /\.(jpe?g|rw2)$/i.test(name);
const roll = dir('CheckRoll', {
'.thumbnails': { hidden: null },
'@eaDir': { junk: null },
'P1.JPG': null,
'2026': { 'P2.JPG': null, '04': { 'P3.RW2': null, 'notes.txt': null } },
'2025': { 'P4.JPG': null },
});
// Walks to the end, one click per pass, which is what the browser does between
// passes: whatever the reader clicked last is what `jump` names.
const walk = async (jumps = []) => {
const pending = [{ dir: roll, rel: '' }];
const out = [];
let pass = 0;
let walked = false;
while (!walked) walked = await walkPass(pending, out, keep, () => jumps[pass++] ?? null);
return { out, passes: pass };
};
// Nothing clicked: the picked folder's own frames, then a layer down, then the
// deepest — a folder is never walked before the folder that holds it.
const { out, passes } = await walk();
const rels = out.map((f) => f.rel);
assert.deepEqual(rels, ['P1.JPG', '2025/P4.JPG', '2026/P2.JPG', '2026/04/P3.RW2'], 'layer by layer from the parent down');
assert.ok(
rels.every((rel, i) => i === 0 || rel.split('/').length >= rels[i - 1].split('/').length),
'no deeper frame is read before a shallower one'
);
assert.ok(!rels.some((rel) => /\.thumbnails|@eaDir|notes\.txt/.test(rel)), 'hidden and unsupported names are skipped');
assert.equal(passes, 4, 'one folder a pass — the thread goes back to the page between them');
// A click on one of the two folders a layer down while its layer is still to be
// walked puts that folder first: the frames the reader is looking at are read
// before the branch beside them.
const clicked = await walk([null, '2026/']);
assert.deepEqual(
clicked.out.map((f) => f.rel),
['P1.JPG', '2026/P2.JPG', '2025/P4.JPG', '2026/04/P3.RW2'],
'the clicked folder jumps the queue'
);
// And the same rule without a browser around it: a pass reads the queue in its
// own order unless the click says otherwise, and a name the queue does not hold
// changes nothing.
const wide = dir('Wide', { a: { 'A.JPG': null }, b: { 'B.JPG': null } });
const before = [];
await walkPass([{ dir: wide, rel: '' }], before, keep, () => null);
assert.deepEqual(before.map((f) => f.rel), ['a/A.JPG', 'b/B.JPG'], 'untouched, the folders are read in the order they were found');
const after = [];
await walkPass([{ dir: wide, rel: '' }], after, keep, () => 'b/');
assert.deepEqual(
after.map((f) => f.rel),
['b/B.JPG', 'a/A.JPG'],
'the folder the reader clicked is read first'
);
assert.equal(jumpQueue([{ rel: 'a/' }], 'nowhere/'), undefined, 'a folder already walked, or never seen, is left alone');
// A pass is bounded: a library wider than the budget is read over several passes,
// and the rest waits in the queue rather than in a loop that never yields.
const children = {};
for (let i = 0; i < DIRS_PER_PASS + 5; i++) children[`d${String(i).padStart(3, '0')}`] = { 'x.jpg': null };
const many = [{ dir: dir('Many', children), rel: '' }];
const found = [];
assert.equal(await walkPass(many, found, keep, () => null), false, 'the queue is not empty after one pass');
assert.equal(found.length, DIRS_PER_PASS - 1, `one pass walked the picked folder and ${DIRS_PER_PASS - 1} below it`);
assert.equal(many.length, 6, 'the rest is left in the queue, for the next pass');
console.log(`ok roll walk — ${rels.length} frames layer by layer, a click jumps the queue, ${DIRS_PER_PASS} folders a pass`);
+37 -5
View File
@@ -75,6 +75,10 @@ export function Library() {
// Rows whose children are folded away. A collapse is a way of looking at the
// tree, not a property of the roll, so it is not remembered between visits.
const [folded, setFolded] = useState<Set<string>>(() => new Set());
// Whether the strip draws the whole branch under the open node or only the
// node itself. Deep is how a roll reads — its top level holds no file of its
// own — and shallow is what a folder of folders is for.
const [deep, setDeep] = useState(true);
const [up, setUp] = useState<string | null>(null);
const [view, setView] = useState<'preview' | 'grid'>('preview');
const [colWidth, setColWidth] = useState(() => {
@@ -89,6 +93,10 @@ export function Library() {
// one more folder.
const [menu, setMenu] = useState<{ folder: LibraryFolder | null; x: number; y: number } | null>(null);
const stopRef = useRef(false);
// The folder the reader last clicked, as the walk spells it ('' for the picked
// folder, `2026/04/` under it): the scan in flight pulls it to the front of
// its queue, so the branch in front of the eye is read first.
const jumpRef = useRef<string | null>(null);
// Where the drag of the column divider started, and how wide the column was
// then — the pointer only ever reports a position, not a distance moved.
const grip = useRef<{ x: number; w: number } | null>(null);
@@ -166,7 +174,7 @@ export function Library() {
setBusy(true);
setNote(null);
try {
const result = await scanFolder(folder, setProgress, () => stopRef.current);
const result = await scanFolder(folder, setProgress, () => stopRef.current, () => jumpRef.current);
setNote(t('lib.scanned', { folder: folder.name, added: result.added, total: result.total }));
await reload();
} catch {
@@ -251,6 +259,12 @@ export function Library() {
const rootName = node ? node.split('/')[0] : null;
// A row's folder as the walk spells it: '' for the picked folder itself, the
// path under it for a subfolder, and nothing for a row belonging to another
// folder — that one is not a folder of this roll.
const relOf = (key: string): string | null =>
!rootName ? null : key === rootName ? '' : key.startsWith(`${rootName}/`) ? `${key.slice(rootName.length + 1)}/` : null;
// The tree: one row per picked folder, then one per subfolder a frame sits in,
// indented by its depth, each carrying how many frames it holds at or under it.
const nodes = useMemo<Node[]>(() => {
@@ -316,13 +330,15 @@ export function Library() {
setNode(nodes.find((n) => n.key === node.split('/')[0])?.key ?? nodes[0]?.key ?? null);
}, [loaded, node, nodes]);
// What the open node draws: its own frames, and the ones in every subfolder
// under it — a roll's top level usually holds no file of its own.
// What the open node draws: its own frames, and — unless the branch has been
// shut out of the strip — the ones in every subfolder under it, because a
// roll's top level usually holds no file of its own.
const shown = useMemo(() => {
if (!rootName) return photos;
const dir = node === rootName ? '' : (node ?? '').slice(rootName.length + 1);
return photos.filter((p) => p.folder === rootName && (dir === '' || p.dir === dir || p.dir.startsWith(`${dir}/`)));
}, [node, photos, rootName]);
const below = dir === '' ? (p: LibraryPhoto) => p.dir !== '' : (p: LibraryPhoto) => p.dir.startsWith(`${dir}/`);
return photos.filter((p) => p.folder === rootName && (p.dir === dir || (deep && below(p))));
}, [deep, node, photos, rootName]);
const frame = shown.find((p) => p.id === up) ?? shown[0] ?? null;
const urls = useThumbUrls(shown);
@@ -415,6 +431,19 @@ export function Library() {
<rect x="9" y="9" width="5" height="5" rx="1" />
</svg>
</button>
{/* Whether the strip draws the branch under the open folder or
only the folder itself. The label says what the click does,
`on` says where the strip is now. */}
<button
type="button"
className={`chip${deep ? ' on' : ''}`}
data-key="lib-subs"
aria-pressed={deep}
title={t('lib.subsHint')}
onClick={() => setDeep((prev) => !prev)}
>
{t(deep ? 'lib.subsHide' : 'lib.subsShow')}
</button>
</>
) : null}
</div>
@@ -465,6 +494,9 @@ export function Library() {
// the rows go.
onClick={() => {
setNode(n.key);
// The folder just clicked is the one a scan in flight
// reads next, whatever layer it sits in.
jumpRef.current = relOf(n.key);
if (!under) return;
setFolded((prev) => {
const next = new Set(prev);
+55 -58
View File
@@ -15,6 +15,7 @@
import type { Recipe } from '../../shared/types';
import { readCapturedAt } from './imageOps';
import { isRawName, rawThumbnail } from './rawDevelop';
import { walkPass, type WalkFile } from './rollWalk';
const DB_NAME = 'recipescam-library';
const FOLDERS = 'folders';
@@ -235,32 +236,13 @@ export interface ScanProgress {
added: number;
}
// The walk goes all the way down: a picked folder is usually a roll of
// date-named subfolders, and a catalogue that only saw its top level would be
// empty. Names beginning with `.` or `@` are skipped — that is `.thumbnails`,
// `.git`, and the `@eaDir` a Synology writes beside every frame.
async function collectFiles(
dir: FileSystemDirectoryHandle,
prefix: string,
out: { handle: FileSystemFileHandle; rel: string }[]
): Promise<void> {
for await (const entry of dir.values()) {
if (/^[.@]/.test(entry.name)) continue;
if (entry.kind === 'directory') {
try {
await collectFiles(entry as FileSystemDirectoryHandle, `${prefix}${entry.name}/`, out);
} catch {
// A subfolder that will not open is skipped; the rest of the roll stands.
}
} else if (isSupportedPhoto(entry.name)) {
out.push({ handle: entry as FileSystemFileHandle, rel: `${prefix}${entry.name}` });
}
}
}
// Walk the folder, keep what is new or changed, and leave the rest alone: a
// second scan of a 2000-file folder only reads the frames that moved.
//
// The walk itself lives in `rollWalk`: layer by layer from the picked folder
// down, one bounded pass at a time, with the folder the reader just clicked
// pulled to the front of the queue. `jump` is that click.
//
// ponytail: read one frame at a time, whole, on the main thread — a RAW is read
// as its bytes for LibRaw and released again. A folder of 5000 RAW files takes
// minutes and makes the page lumpy while it runs. Move the walk into a worker
@@ -268,12 +250,13 @@ async function collectFiles(
export async function scanFolder(
folder: LibraryFolder,
onProgress?: (p: ScanProgress) => void,
shouldStop?: () => boolean
shouldStop?: () => boolean,
jump?: () => string | null
): Promise<ScanProgress> {
const known = new Map((await listPhotos(folder.name)).map((p) => [p.id, p]));
const entries: { handle: FileSystemFileHandle; rel: string }[] = [];
await collectFiles(folder.handle, '', entries);
const progress: ScanProgress = { folder: folder.name, total: entries.length, done: 0, added: 0 };
const pending = [{ dir: folder.handle, rel: '' }];
const entries: WalkFile[] = [];
const progress: ScanProgress = { folder: folder.name, total: 0, done: 0, added: 0 };
let batch: LibraryPhoto[] = [];
// One transaction per batch, a put per row: a store with `keyPath: 'id'` takes
// a record, not an array of them.
@@ -290,39 +273,53 @@ export async function scanFolder(
tx.onerror = () => reject(tx.error);
});
};
for (const { handle, rel } of entries) {
if (shouldStop?.()) break;
progress.done++;
try {
const file = await handle.getFile();
const id = photoId(folder.name, rel);
const seen = known.get(id);
if (seen && seen.size === file.size && seen.taken && seen.taken === file.lastModified && seen.thumb) {
onProgress?.(progress);
continue;
let stop = false;
let walked = false;
// One bounded pass at a time, and the pass yields between frames, so a folder
// clicked while the scan runs is walked on the next jump rather than after the
// whole tree. Nothing clicked: parents first, then each layer below, deepest
// last — the order `pending` is filled in.
do {
const from = entries.length;
walked = await walkPass(pending, entries, isSupportedPhoto, jump ?? (() => null));
progress.total += entries.length - from;
for (const { handle, rel } of entries.slice(from)) {
if (shouldStop?.()) {
stop = true;
break;
}
const bytes = new Uint8Array(await file.arrayBuffer());
const [thumb, taken] = [await makeThumbnail(file, bytes), (await readCapturedAt(bytes)) ?? file.lastModified];
const cut = rel.lastIndexOf('/');
batch.push({
id,
folder: folder.name,
dir: cut < 0 ? '' : rel.slice(0, cut),
name: handle.name,
handle,
thumb,
taken,
size: file.size,
addedAt: seen?.addedAt ?? Date.now(),
});
progress.added++;
if (batch.length >= BATCH) await flush();
} catch {
// A frame that will not read is a frame the catalogue skips: one bad file
// in a folder is not a failed folder.
progress.done++;
try {
const file = await handle.getFile();
const id = photoId(folder.name, rel);
const seen = known.get(id);
if (seen && seen.size === file.size && seen.taken && seen.taken === file.lastModified && seen.thumb) {
onProgress?.(progress);
continue;
}
const bytes = new Uint8Array(await file.arrayBuffer());
const [thumb, taken] = [await makeThumbnail(file, bytes), (await readCapturedAt(bytes)) ?? file.lastModified];
const cut = rel.lastIndexOf('/');
batch.push({
id,
folder: folder.name,
dir: cut < 0 ? '' : rel.slice(0, cut),
name: handle.name,
handle,
thumb,
taken,
size: file.size,
addedAt: seen?.addedAt ?? Date.now(),
});
progress.added++;
if (batch.length >= BATCH) await flush();
} catch {
// A frame that will not read is a frame the catalogue skips: one bad file
// in a folder is not a failed folder.
}
onProgress?.(progress);
}
onProgress?.(progress);
}
} while (!walked && !stop);
await flush();
return progress;
}
+68
View File
@@ -0,0 +1,68 @@
// Walking a picked folder is the one part of the catalogue that can run for
// minutes: a roll is dated folders inside dated folders, and a library can hold
// thousands. The walk goes from the parent down, one layer at a time, so the
// frames nearest the top are indexed first — and a folder the reader clicks is
// put in front of the queue, so the branch in front of them is read before the
// layer it sits in would have come round. A pass covers `DIRS_PER_PASS`
// directories and then hands the thread back, which is what keeps the page
// answering while a scan runs.
//
// Pure on purpose: it only reads `values()` off a directory handle, so a check
// can walk a stand-in tree without a browser (scripts/roll-walk-check.mjs).
export interface WalkFile {
handle: FileSystemFileHandle;
rel: string;
}
interface Pending {
dir: FileSystemDirectoryHandle;
rel: string;
}
// Directories one pass walks before it yields: small enough that a click is
// answered within a second, large enough that a folder of a few hundred is one
// pass rather than a hundred.
export const DIRS_PER_PASS = 200;
// The folder the reader clicked goes first. `rel` is how the queue spells a
// folder — '' for the picked one, `2026/04/` under it — and a folder already
// walked is simply not in the queue, which is the no-op case.
//
// ponytail: only a folder the queue already holds can jump, which is every
// folder the tree can draw, because a row exists per indexed frame. A tree that
// listed folders before their first frame would need `getDirectoryHandle` to
// descend to the clicked path here.
export function jumpQueue(pending: Pending[], rel: string | null): void {
if (rel === null) return;
const at = pending.findIndex((p) => p.rel === rel);
if (at > 0) pending.unshift(pending.splice(at, 1)[0]);
}
// One pass over the queue. The queue is the walk's own state: what is behind has
// been walked, what is ahead is in the order it was found — parents before
// children, shallower layers before deeper ones. Returns true when nothing is
// left to walk.
export async function walkPass(
pending: Pending[],
out: WalkFile[],
keep: (name: string) => boolean,
jump: () => string | null
): Promise<boolean> {
for (let i = 0; i < DIRS_PER_PASS && pending.length; i++) {
jumpQueue(pending, jump());
const here = pending.shift()!;
try {
for await (const entry of here.dir.values()) {
// Names beginning with `.` or `@` are skipped — that is `.thumbnails`,
// `.git`, and the `@eaDir` a Synology writes beside every frame.
if (/^[.@]/.test(entry.name)) continue;
if (entry.kind === 'directory') pending.push({ dir: entry as FileSystemDirectoryHandle, rel: `${here.rel}${entry.name}/` });
else if (keep(entry.name)) out.push({ handle: entry as FileSystemFileHandle, rel: `${here.rel}${entry.name}` });
}
} catch {
// A subfolder that will not open is skipped; the rest of the roll stands.
}
}
return pending.length === 0;
}
+3
View File
@@ -240,6 +240,9 @@ export const en: Dict = {
'lib.colWidth': 'Drag to set the width of the folder column (arrow keys when focused)',
'lib.viewPreview': 'SINGLE',
'lib.viewGrid': 'ALL THUMBS',
'lib.subsShow': 'SHOW SUBFOLDERS',
'lib.subsHide': 'THIS FOLDER ONLY',
'lib.subsHint': 'Frames in subfolders: the whole branch under the open folder, or only what that folder holds itself.',
'lib.open': 'Open in the studio',
'lib.edited': 'edited',
'lib.size': '{mb} MB',
+4
View File
@@ -253,6 +253,10 @@ export const vi = {
'lib.colWidth': 'Kéo để chỉnh độ rộng cột thư mục (mũi tên trái/phải khi đang chọn)',
'lib.viewPreview': 'MỘT ẢNH',
'lib.viewGrid': 'TOÀN BỘ',
'lib.subsShow': 'HIỆN CẢ THƯ MỤC CON',
'lib.subsHide': 'CHỈ THƯ MỤC ĐANG CHỌN',
'lib.subsHint':
'Ảnh trong thư mục con: hiện cả nhánh dưới thư mục đang chọn, hoặc chỉ ảnh của chính thư mục đó.',
'lib.open': 'Mở trong studio',
'lib.edited': 'đã chỉnh',
'lib.size': '{mb} MB',