3dtours/RecipesCam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

web: name a roll's folders before reading their frames

Browse Source
This commit is contained in:
3dtours
2026-09-28 20:06:10 +07:00
parent d0fddba1e8
commit 52732a17ee
5 changed files with 300 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/frontend/src/Library.tsx
+44 -15
View File
@@ -6,12 +6,14 @@ import {
getPhoto,
listEditedIds,
listFolders,
listDirs,
listPhotos,
pickFolder,
readPhotoFile,
removeFolder,
renameFolder,
scanFolder,
type LibraryDir,
type LibraryFolder,
type LibraryPhoto,
type ScanProgress,
@@ -67,6 +69,9 @@ export function Library() {
const [folders, setFolders] = useState<LibraryFolder[]>([]);
const [ready, setReady] = useState<Set<string>>(new Set());
const [photos, setPhotos] = useState<LibraryPhoto[]>([]);
// The folders a scan has walked into. A roll's own shape is on screen from the
// first pass, before the frames under its folders have been read.
const [dirs, setDirs] = useState<LibraryDir[]>([]);
const [edited, setEdited] = useState<Set<string>>(new Set());
// Which tree node is open, which frame is up, and which of the two middle
// views (the frame, or every thumbnail of the open node) is on. The open node
@@ -106,9 +111,10 @@ export function Library() {
const [loaded, setLoaded] = useState(false);
const reload = useCallback(async () => {
const [rows, edits] = await Promise.all([listPhotos(), listEditedIds()]);
const [rows, edits, dirs] = await Promise.all([listPhotos(), listEditedIds(), listDirs()]);
setPhotos(rows);
setEdited(edits);
setDirs(dirs);
}, []);
// The catalogue outlives the tab; the permission to read those folders does
@@ -174,7 +180,23 @@ export function Library() {
setBusy(true);
setNote(null);
try {
const result = await scanFolder(folder, setProgress, () => stopRef.current, () => jumpRef.current);
// The column grows while the roll is read: the folders the walk has named
// are on screen a pass before the frames under them. A refresh reads the
// whole catalogue, so it keeps to one every few hundred milliseconds
// rather than one per frame.
let last = 0;
const result = await scanFolder(
folder,
(p) => {
setProgress(p);
if (p.done === 0 || Date.now() - last > 700) {
last = Date.now();
void reload();
}
},
() => stopRef.current,
() => jumpRef.current
);
setNote(t('lib.scanned', { folder: folder.name, added: result.added, total: result.total }));
await reload();
} catch {
@@ -265,30 +287,37 @@ export function Library() {
const relOf = (key: string): string | null =>
!rootName ? null : key === rootName ? '' : key.startsWith(`${rootName}/`) ? `${key.slice(rootName.length + 1)}/` : null;
// The tree: one row per picked folder, then one per subfolder a frame sits in,
// indented by its depth, each carrying how many frames it holds at or under it.
// The tree: one row per picked folder, then one per subfolder, indented by its
// depth — the folders the walk has named, whether or not a frame has been read
// out of them, each carrying how many frames sit at or under it.
const nodes = useMemo<Node[]>(() => {
const hits = new Map<string, number>();
const dirs = new Map<string, Set<string>>();
for (const photo of photos) {
hits.set(photo.folder, (hits.get(photo.folder) ?? 0) + 1);
if (!photo.dir) continue;
const set = dirs.get(photo.folder) ?? new Set<string>();
dirs.set(photo.folder, set);
const foldersUnder = new Map<string, Set<string>>();
// One path per folder between the picked one and `rel`, so a row is drawn for
// every folder on the way to a frame as well as for the frame's own.
const add = (folder: string, rel: string, counted: boolean) => {
const set = foldersUnder.get(folder) ?? new Set<string>();
foldersUnder.set(folder, set);
let path = '';
for (const part of photo.dir.split('/')) {
for (const part of rel.split('/')) {
path = path ? `${path}/${part}` : part;
set.add(path);
hits.set(`${photo.folder}/${path}`, (hits.get(`${photo.folder}/${path}`) ?? 0) + 1);
if (counted) hits.set(`${folder}/${path}`, (hits.get(`${folder}/${path}`) ?? 0) + 1);
}
};
for (const photo of photos) {
hits.set(photo.folder, (hits.get(photo.folder) ?? 0) + 1);
if (photo.dir) add(photo.folder, photo.dir, true);
}
// A folder the walk found is a row before it holds a frame: the names arrive
// a pass ahead of the thumbnails.
for (const dir of dirs) add(dir.folder, dir.rel, false);
const out: Node[] = [];
for (const folder of folders) {
out.push({ key: folder.name, label: folder.label || folder.name, depth: 0, count: hits.get(folder.name) ?? 0 });
// Path order puts a folder before its own children, which is the order the
// indented rows have to come in.
const own = [...(dirs.get(folder.name) ?? [])].sort((a, b) => a.localeCompare(b));
for (const dir of own) {
for (const dir of [...(foldersUnder.get(folder.name) ?? [])].sort((a, b) => a.localeCompare(b))) {
const parts = dir.split('/');
out.push({
key: `${folder.name}/${dir}`,
@@ -299,7 +328,7 @@ export function Library() {
}
}
return out;
}, [folders, photos]);
}, [dirs, folders, photos]);
// A row with anything under it is the one that folds: its key is the prefix of
// its children's, in the path order the rows come in.
docker/frontend/src/engine/library.ts
+86 -17
View File
@@ -15,12 +15,15 @@
import type { Recipe } from '../../shared/types';
import { readCapturedAt } from './imageOps';
import { isRawName, rawThumbnail } from './rawDevelop';
import { walkPass, type WalkFile } from './rollWalk';
import { walkPass, type Walk, type WalkFile } from './rollWalk';
const DB_NAME = 'recipescam-library';
const FOLDERS = 'folders';
const PHOTOS = 'photos';
const EDITS = 'edits';
// The folders a scan has walked into. They are names before they are frames: the
// column draws the shape of a roll while the frames are still being read.
const DIRS = 'dirs';
// The grid cell is ~220px wide and a retina display doubles it: 512 on the long
// edge is the largest a tile ever shows, and it costs ~30KB per frame.
@@ -56,6 +59,16 @@ export interface LibraryPhoto {
addedAt: number;
}
// A folder the walk has found, whether or not a frame has been read out of it
// yet: the column draws a roll from these before its frames arrive.
export interface LibraryDir {
// `${folder}/${rel}` — the tree key, the same way a frame id is spelled.
id: string;
folder: string;
// The path under the picked folder, no leading or trailing slash.
rel: string;
}
export interface LibraryEdit {
photoId: string;
recipe: Recipe;
@@ -83,14 +96,19 @@ let dbPromise: Promise<IDBDatabase> | null = null;
function openDb(): Promise<IDBDatabase> {
dbPromise ??= new Promise<IDBDatabase>((resolve, reject) => {
const req = indexedDB.open(DB_NAME, 1);
const req = indexedDB.open(DB_NAME, 2);
req.onupgradeneeded = () => {
const db = req.result;
db.createObjectStore(FOLDERS, { keyPath: 'name' });
const photos = db.createObjectStore(PHOTOS, { keyPath: 'id' });
photos.createIndex('folder', 'folder');
photos.createIndex('taken', 'taken');
db.createObjectStore(EDITS, { keyPath: 'photoId' });
if (!db.objectStoreNames.contains(FOLDERS)) db.createObjectStore(FOLDERS, { keyPath: 'name' });
if (!db.objectStoreNames.contains(PHOTOS)) {
const photos = db.createObjectStore(PHOTOS, { keyPath: 'id' });
photos.createIndex('folder', 'folder');
photos.createIndex('taken', 'taken');
}
if (!db.objectStoreNames.contains(EDITS)) db.createObjectStore(EDITS, { keyPath: 'photoId' });
if (!db.objectStoreNames.contains(DIRS)) {
db.createObjectStore(DIRS, { keyPath: 'id' }).createIndex('folder', 'folder');
}
};
req.onsuccess = () => resolve(req.result);
req.onerror = () => reject(req.error);
@@ -145,15 +163,18 @@ export async function renameFolder(folder: LibraryFolder, label: string): Promis
export async function removeFolder(name: string): Promise<void> {
const ids = (await listPhotos(name)).map((p) => p.id);
const dirs = await listDirs(name);
const db = await openDb();
await new Promise<void>((resolve, reject) => {
const tx = db.transaction([FOLDERS, PHOTOS, EDITS], 'readwrite');
const tx = db.transaction([FOLDERS, PHOTOS, EDITS, DIRS], 'readwrite');
tx.objectStore(FOLDERS).delete(name);
const photos = tx.objectStore(PHOTOS);
for (const id of ids) {
photos.delete(id);
tx.objectStore(EDITS).delete(id);
}
const store = tx.objectStore(DIRS);
for (const dir of dirs) store.delete(dir.id);
tx.oncomplete = () => resolve();
tx.onerror = () => reject(tx.error);
});
@@ -254,21 +275,31 @@ export async function scanFolder(
jump?: () => string | null
): Promise<ScanProgress> {
const known = new Map((await listPhotos(folder.name)).map((p) => [p.id, p]));
const pending = [{ dir: folder.handle, rel: '' }];
const tree: Walk = { root: folder.handle, pending: [{ dir: folder.handle, rel: '' }], walked: new Set() };
const entries: WalkFile[] = [];
// What the passes found: the folders this scan walked into, and the names they
// put in the column. `names` is cleared a pass at a time; `found` keeps the
// whole scan so a folder that has gone from the disk goes from the column too.
const names: string[] = [];
const found = new Set<string>();
const progress: ScanProgress = { folder: folder.name, total: 0, done: 0, added: 0 };
let batch: LibraryPhoto[] = [];
let dirs: LibraryDir[] = [];
// One transaction per batch, a put per row: a store with `keyPath: 'id'` takes
// a record, not an array of them.
const flush = async () => {
if (!batch.length) return;
if (!batch.length && !dirs.length) return;
const rows = batch;
const folders = dirs;
batch = [];
dirs = [];
const db = await openDb();
await new Promise<void>((resolve, reject) => {
const tx = db.transaction(PHOTOS, 'readwrite');
const store = tx.objectStore(PHOTOS);
for (const row of rows) store.put(row);
const tx = db.transaction([PHOTOS, DIRS], 'readwrite');
const photos = tx.objectStore(PHOTOS);
for (const row of rows) photos.put(row);
const store = tx.objectStore(DIRS);
for (const dir of folders) store.put(dir);
tx.oncomplete = () => resolve();
tx.onerror = () => reject(tx.error);
});
@@ -276,12 +307,22 @@ export async function scanFolder(
let stop = false;
let walked = false;
// One bounded pass at a time, and the pass yields between frames, so a folder
// clicked while the scan runs is walked on the next jump rather than after the
// whole tree. Nothing clicked: parents first, then each layer below, deepest
// last — the order `pending` is filled in.
// clicked while the scan runs is read on the next jump rather than after the
// whole tree. Nothing clicked: the picked folder and its frames, then each
// layer below with theirs, deepest last — the order `pending` is filled in.
do {
const from = entries.length;
walked = await walkPass(pending, entries, isSupportedPhoto, jump ?? (() => null));
names.length = 0;
walked = await walkPass(tree, entries, names, isSupportedPhoto, jump ?? (() => null));
// The names come in before the frames they hold: the column grows one pass
// ahead of the strip, which is the whole point of reading layer by layer.
for (const rel of names) {
const id = `${folder.name}/${rel.slice(0, -1)}`;
if (found.has(id)) continue;
found.add(id);
dirs.push({ id, folder: folder.name, rel: rel.slice(0, -1) });
}
if (dirs.length >= BATCH) await flush();
progress.total += entries.length - from;
for (const { handle, rel } of entries.slice(from)) {
if (shouldStop?.()) {
@@ -321,9 +362,27 @@ export async function scanFolder(
}
} while (!walked && !stop);
await flush();
// A stopped scan has not seen the whole roll, so only a walk that reached the
// end may say which folders are gone.
if (walked) await pruneDirs(folder.name, found);
return progress;
}
// The folders that are no longer under the picked one: a rescan of a roll that
// has been tidied on the disk takes the tidied rows out of the column with it.
async function pruneDirs(folder: string, keep: Set<string>): Promise<void> {
const gone = (await listDirs(folder)).filter((d) => !keep.has(d.id));
if (!gone.length) return;
const db = await openDb();
await new Promise<void>((resolve, reject) => {
const tx = db.transaction(DIRS, 'readwrite');
const store = tx.objectStore(DIRS);
for (const dir of gone) store.delete(dir.id);
tx.oncomplete = () => resolve();
tx.onerror = () => reject(tx.error);
});
}
// --- reading ---------------------------------------------------------------
export async function listPhotos(folder?: string): Promise<LibraryPhoto[]> {
@@ -338,6 +397,16 @@ export async function listPhotos(folder?: string): Promise<LibraryPhoto[]> {
}
}
// The folders a scan has walked into, whether or not a frame has been read out
// of them yet — the column draws these as soon as the parent pass named them.
export async function listDirs(folder?: string): Promise<LibraryDir[]> {
try {
return await ask<LibraryDir[]>(DIRS, 'readonly', (s) => (folder ? s.index('folder').getAll(folder) : s.getAll()));
} catch {
return [];
}
}
export async function getPhoto(id: string): Promise<LibraryPhoto | null> {
try {
return (await ask<LibraryPhoto | undefined>(PHOTOS, 'readonly', (s) => s.get(id))) ?? null;
docker/frontend/src/engine/rollWalk.ts
+64 -31
View File
@@ -1,14 +1,17 @@
// Walking a picked folder is the one part of the catalogue that can run for
// minutes: a roll is dated folders inside dated folders, and a library can hold
// thousands. The walk goes from the parent down, one layer at a time, so the
// frames nearest the top are indexed first — and a folder the reader clicks is
// put in front of the queue, so the branch in front of them is read before the
// layer it sits in would have come round. A pass covers `DIRS_PER_PASS`
// directories and then hands the thread back, which is what keeps the page
// answering while a scan runs.
// thousands. The walk goes from the parent down, one layer at a time — the picked
// folder and its own frames, then every folder one level down with theirs, then
// the level below that — so the names in the column and the frames near the top
// are on screen within the first pass. A folder the reader clicks is opened
// straight away and put in front of the queue, so the branch in front of them is
// read before the layer it sits in would have come round. A pass covers
// `DIRS_PER_PASS` directories and then hands the thread back, which is what keeps
// the page answering while a scan runs.
//
// Pure on purpose: it only reads `values()` off a directory handle, so a check
// can walk a stand-in tree without a browser (scripts/roll-walk-check.mjs).
// Pure on purpose: it only reads `values()` and `getDirectoryHandle()` off a
// directory handle, so a check can walk a stand-in tree without a browser
// (scripts/roll-walk-check.mjs).
export interface WalkFile {
handle: FileSystemFileHandle;
@@ -20,49 +23,79 @@ interface Pending {
rel: string;
}
// Directories one pass walks before it yields: small enough that a click is
// What the walk is: the folder everything is measured from, the directories still
// to read, and the ones already read. `rel` is how a folder is spelled here — ''
// for the picked one, `2026/04/` under it, always with the trailing slash.
export interface Walk {
root: FileSystemDirectoryHandle;
pending: Pending[];
walked: Set<string>;
}
// Directories one pass reads before it yields: small enough that a click is
// answered within a second, large enough that a folder of a few hundred is one
// pass rather than a hundred.
export const DIRS_PER_PASS = 200;
// The folder the reader clicked goes first. `rel` is how the queue spells a
// folder — '' for the picked one, `2026/04/` under it — and a folder already
// walked is simply not in the queue, which is the no-op case.
//
// ponytail: only a folder the queue already holds can jump, which is every
// folder the tree can draw, because a row exists per indexed frame. A tree that
// listed folders before their first frame would need `getDirectoryHandle` to
// descend to the clicked path here.
export function jumpQueue(pending: Pending[], rel: string | null): void {
if (rel === null) return;
const at = pending.findIndex((p) => p.rel === rel);
if (at > 0) pending.unshift(pending.splice(at, 1)[0]);
// The folder the reader clicked goes first. When it is not in the queue the walk
// has not reached it yet — its row is on screen because the parent pass named it
// — so it is opened from the picked folder and read next. A folder already read,
// or one that will not open, changes nothing.
export async function jumpQueue(walk: Walk, rel: string | null): Promise<void> {
if (!rel || walk.walked.has(rel)) return;
const at = walk.pending.findIndex((p) => p.rel === rel);
if (at >= 0) {
if (at > 0) walk.pending.unshift(walk.pending.splice(at, 1)[0]);
return;
}
let dir = walk.root;
for (const part of rel.split('/').filter(Boolean)) {
try {
dir = await dir.getDirectoryHandle(part);
} catch {
return;
}
}
// ponytail: the reader's folder jumps the whole queue, ancestors included —
// the folders between it and the picked one are read when their layer comes.
walk.pending.unshift({ dir, rel });
}
// One pass over the queue. The queue is the walk's own state: what is behind has
// been walked, what is ahead is in the order it was found — parents before
// children, shallower layers before deeper ones. Returns true when nothing is
// left to walk.
// been read, what is ahead is in the order it was found — parents before
// children, shallower layers before deeper ones. `seen` collects the folders this
// pass found, which is what the column draws before they hold a frame. Returns
// true when nothing is left to walk.
export async function walkPass(
pending: Pending[],
walk: Walk,
out: WalkFile[],
seen: string[],
keep: (name: string) => boolean,
jump: () => string | null
): Promise<boolean> {
for (let i = 0; i < DIRS_PER_PASS && pending.length; i++) {
jumpQueue(pending, jump());
const here = pending.shift()!;
for (let i = 0; i < DIRS_PER_PASS && walk.pending.length; i++) {
await jumpQueue(walk, jump());
const here = walk.pending.shift()!;
// A folder opened by a click is still in the queue its parent filled: it is
// read once, in the place the click put it.
if (walk.walked.has(here.rel)) continue;
walk.walked.add(here.rel);
try {
for await (const entry of here.dir.values()) {
// Names beginning with `.` or `@` are skipped — that is `.thumbnails`,
// `.git`, and the `@eaDir` a Synology writes beside every frame.
if (/^[.@]/.test(entry.name)) continue;
if (entry.kind === 'directory') pending.push({ dir: entry as FileSystemDirectoryHandle, rel: `${here.rel}${entry.name}/` });
else if (keep(entry.name)) out.push({ handle: entry as FileSystemFileHandle, rel: `${here.rel}${entry.name}` });
if (entry.kind === 'directory') {
const rel = `${here.rel}${entry.name}/`;
walk.pending.push({ dir: entry as FileSystemDirectoryHandle, rel });
seen.push(rel);
} else if (keep(entry.name)) {
out.push({ handle: entry as FileSystemFileHandle, rel: `${here.rel}${entry.name}` });
}
}
} catch {
// A subfolder that will not open is skipped; the rest of the roll stands.
}
}
return pending.length === 0;
return walk.pending.length === 0;
}