diff --git a/docker/frontend/scripts/dup-check.mjs b/docker/frontend/scripts/dup-check.mjs new file mode 100644 index 0000000..255725b --- /dev/null +++ b/docker/frontend/scripts/dup-check.mjs @@ -0,0 +1,218 @@ +// TEMP repro: UPDATE on a folder row must not write the frames it already holds +// a second time. Root handle is named `\` — a Windows drive root, which is what +// the reader right-clicked. +import { spawn } from 'node:child_process'; +import { createServer } from 'node:http'; +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const FRONTEND = fileURLToPath(new URL('..', import.meta.url)); +const SAMPLE_FILE = process.env.SAMPLE_FILE ?? path.resolve(FRONTEND, '../sample/_3279791-develop-check.jpg'); +const APP_PORT = Number(process.env.APP_PORT ?? 5920); +const SAMPLE_PORT = Number(process.env.SAMPLE_PORT ?? 5921); +const ROOT = '\\'; +const ROOT_KEY = process.env.ROOT_KEY ?? ROOT; +const names = ['a1.jpg', 'a2.jpg', 'a3.jpg', 'a4.jpg']; +const inner = ['b1.jpg', 'b2.jpg']; + +let fails = 0; +const step = (what, ok, detail = '') => { + if (!ok) fails++; + console.log(`${ok ? 'ok ' : 'FAIL'} ${what}${detail ? ` — ${detail}` : ''}`); +}; + +const playwright = await import(process.env.PLAYWRIGHT_CORE ?? 'playwright-core').catch(() => null); +if (!playwright) { + console.log('SKIP playwright-core is not installed'); + process.exit(0); +} + +const jpg = await readFile(SAMPLE_FILE); +const all = [...names, ...inner]; +const sampleServer = createServer((req, res) => { + const name = path.basename(decodeURIComponent((req.url ?? '/').split('?')[0])); + if (!all.includes(name)) { + res.writeHead(404).end(); + return; + } + res.writeHead(200, { 'Content-Type': 'image/jpeg', 'Content-Length': jpg.length, 'Access-Control-Allow-Origin': '*' }); + res.end(jpg); +}); +await new Promise((resolve) => sampleServer.listen(SAMPLE_PORT, '127.0.0.1', resolve)); + +const app = process.env.BASE_URL + ? null + : spawn('npx', ['vite', 'preview', '--port', String(APP_PORT), '--strictPort', '--host', '127.0.0.1'], { + cwd: FRONTEND, + stdio: 'ignore', + detached: true, + }); +const stopApp = () => { + if (!app?.pid) return; + try { + process.kill(-app.pid, 'SIGTERM'); + } catch {} +}; +const base = process.env.BASE_URL ?? `http://127.0.0.1:${APP_PORT}`; +let up = !app; +for (let i = 0; i < 60 && !up; i++) { + up = await fetch(`${base}/library`).then((r) => r.ok).catch(() => false); + if (!up) await new Promise((r) => setTimeout(r, 500)); +} +if (!up) { + stopApp(); + sampleServer.close(); + console.log(`FAIL no preview on ${base}`); + process.exit(1); +} + +const browser = await playwright.chromium.launch({ + executablePath: process.env.CHROME || undefined, + args: ['--no-sandbox', '--enable-unsafe-swiftshader'], +}); +const context = await browser.newContext({ viewport: { width: 1500, height: 950 } }); + +await context.route('**/api/**', (route) => { + const url = route.request().url(); + const json = (body) => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(body) }); + if (url.includes('/auth/me')) return json({ user: { id: 1, email: 'check@example.com', name: 'check', verified: true, pro: true, admin: false } }); + if (url.includes('/photos')) return json({ photos: [] }); + if (url.includes('/place')) return json({ place: 'Hanoi' }); + if (url.includes('/ratings')) return json({ ratings: [] }); + if (url.includes('/highlights')) return json({ highlights: { day: [], week: [], ever: [] } }); + if (url.includes('/recipes')) return json({ recipes: [] }); + return json({}); +}); + +await context.addInitScript( + ({ origin, names, inner, root }) => { + const fileHandle = (name) => { + const handle = { kind: 'file', name, __fake: true }; + Object.defineProperty(handle, 'getFile', { + value: async () => new File([await (await fetch(`${origin}/${name}`)).arrayBuffer()], name, { type: 'image/jpeg', lastModified: Date.UTC(2024, 3, 2, 12) }), + }); + return handle; + }; + const valuesOf = (node, label) => { + const it = []; + for (const f of node.files) it.push(fileHandle(f)); + for (const d of node.dirs ?? []) it.push(dirHandle(d.label, d, label)); + return it[Symbol.iterator](); + }; + const dirHandle = (label, node, parent = '') => { + const handle = { kind: 'directory', name: label, __fake: true }; + const self = () => dirHandle(label, node, parent); + Object.defineProperties(handle, { + values: { value: () => valuesOf(node, label) }, + queryPermission: { value: async () => 'granted' }, + requestPermission: { value: async () => 'granted' }, + getDirectoryHandle: { + value: async (name) => { + const d = (node.dirs ?? []).find((x) => x.label === name); + if (!d) throw new Error(`no folder ${name}`); + return dirHandle(name, d, label); + }, + }, + getFileHandle: { + value: async (name) => { + if (!node.files.includes(name)) throw new Error(`no frame ${name}`); + return fileHandle(name); + }, + }, + __self: { value: self }, + }); + return handle; + }; + const rootNode = { files: names, dirs: [{ label: 'Early', files: inner }] }; + const rootHandle = () => dirHandle(root, rootNode); + window.showDirectoryPicker = async () => rootHandle(); + + const revive = (row) => { + if (row?.handle?.__fake && !Object.getOwnPropertyDescriptor(row.handle, 'getFile')) { + row.handle = row.handle.kind === 'directory' ? rootHandle() : fileHandle(row.handle.name); + } + return row; + }; + for (const method of ['get', 'getAll']) { + const original = IDBObjectStore.prototype[method]; + IDBObjectStore.prototype[method] = function (...args) { + const request = original.apply(this, args); + const descriptor = Object.getOwnPropertyDescriptor(IDBRequest.prototype, 'result'); + Object.defineProperty(request, 'result', { + configurable: true, + get() { + const value = descriptor.get.call(this); + return Array.isArray(value) ? value.map(revive) : revive(value); + }, + }); + return request; + }; + } + }, + { origin: `http://127.0.0.1:${SAMPLE_PORT}`, names, inner, root: ROOT } +); + +const page = await context.newPage(); +const errors = []; +page.on('pageerror', (e) => errors.push(String(e.message).slice(0, 200))); + +const rows = () => + page.evaluate( + (db) => + new Promise((resolve) => { + const open = indexedDB.open(db); + open.onsuccess = () => { + const tx = open.result.transaction('photos', 'readonly'); + const req = tx.objectStore('photos').getAll(); + req.onsuccess = () => { + const photos = req.result.map((r) => ({ id: r.id, folder: r.folder, dir: r.dir, name: r.name, size: r.size, mtime: r.mtime })); + const t = open.result.transaction('thumbs', 'readonly'); + const tr = t.objectStore('thumbs').getAll(); + tr.onsuccess = () => + resolve({ photos, thumbs: tr.result.map((x) => x.id), stores: [...open.result.objectStoreNames] }); + }; + }; + open.onerror = () => resolve({ photos: [], thumbs: [], stores: [] }); + }), + 'recipescam-library' + ); + +await page.goto(`${base}/library`); +await page.click('[data-key="lib-add"]'); +await page.waitForSelector('[data-key^="lib-node-"]', { timeout: 60_000 }); +await page + .waitForFunction((n) => (document.querySelector('[data-key="lib-count"]')?.textContent ?? '').includes(String(n)), all.length, { timeout: 240_000, polling: 250 }) + .catch(() => {}); +const first = await rows(); +step('the drive is read once', first.photos.length === all.length, `${first.photos.length} rows for ${all.length} frames`); +step('every frame has one row', new Set(first.photos.map((p) => p.id)).size === first.photos.length, first.photos.map((p) => p.id).join(' | ')); + +// The reader right-clicks the drive row and asks for UPDATE. +await page.locator('[data-key^="lib-node-"]').first().click({ button: 'right' }); +await page.waitForSelector('[data-key^="lib-update-"]', { timeout: 20_000 }); +await page.click('[data-key^="lib-update-"]'); +await page.waitForFunction(() => !document.querySelector('.lib-scan-live, [data-key="lib-scan-stop"]'), { timeout: 240_000, polling: 250 }).catch(() => {}); +await page.waitForTimeout(4000); + +const after = await rows(); +const byId = new Map(); +for (const p of after.photos) byId.set(p.id, (byId.get(p.id) ?? 0) + 1); +const dupIds = [...byId].filter(([, n]) => n > 1); +const byName = new Map(); +for (const p of after.photos) byName.set(p.name, (byName.get(p.name) ?? 0) + 1); +const dupNames = [...byName].filter(([, n]) => n > 1); +step('UPDATE adds nothing', after.photos.length === all.length, `${after.photos.length} rows for ${all.length} frames`); +step('no frame carries two rows', dupIds.length === 0, dupIds.map(([id, n]) => `${id}×${n}`).join(' | ')); +step('no frame carries two thumbnails', dupNames.length === 0 && after.thumbs.length === all.length, `rows=${after.photos.length} thumbs=${after.thumbs.length} | ${dupNames.map(([k, n]) => `${k}×${n}`).join(' ')}`); +const tiles = await page.evaluate(() => document.querySelectorAll('.lib-tile, .mob-lib-thumb-cell').length); +step('the wall draws each frame once', tiles <= all.length + 2, `tiles=${tiles}`); +step('no page error', errors.length === 0, errors.join(' | ')); + +console.log('ids after:', after.photos.map((p) => `${p.id} [folder=${p.folder}]`).join(' | ')); + +await browser.close(); +stopApp(); +sampleServer.close(); +console.log(fails ? `${fails} FAIL` : 'all ok'); +process.exit(fails ? 1 : 0); diff --git a/docker/frontend/scripts/legacy-dup-check.mjs b/docker/frontend/scripts/legacy-dup-check.mjs new file mode 100644 index 0000000..187025e --- /dev/null +++ b/docker/frontend/scripts/legacy-dup-check.mjs @@ -0,0 +1,241 @@ +// TEMP repro: a drive root whose catalogue holds one frame twice — the row an +// older build filed under its raw `\`-spelled id, and the row this build filed +// beside it — must come out of UPDATE as one row, and the frame keeps the id +// this build computes. This is the state the reader was left in: two tiles for +// every frame of the disk, and an UPDATE that only ever added more. +import { spawn } from 'node:child_process'; +import { createServer } from 'node:http'; +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const FRONTEND = fileURLToPath(new URL('..', import.meta.url)); +const SAMPLE_FILE = process.env.SAMPLE_FILE ?? path.resolve(FRONTEND, '../sample/_3279791-develop-check.jpg'); +const APP_PORT = Number(process.env.APP_PORT ?? 5930); +const SAMPLE_PORT = Number(process.env.SAMPLE_PORT ?? 5931); +const ROOT = '\\'; +const names = ['a1.jpg', 'a2.jpg', 'a3.jpg', 'a4.jpg']; + +let fails = 0; +const step = (what, ok, detail = '') => { + if (!ok) fails++; + console.log(`${ok ? 'ok ' : 'FAIL'} ${what}${detail ? ` — ${detail}` : ''}`); +}; + +const playwright = await import(process.env.PLAYWRIGHT_CORE ?? 'playwright-core').catch(() => null); +if (!playwright) { + console.log('SKIP playwright-core is not installed'); + process.exit(0); +} + +const jpg = await readFile(SAMPLE_FILE); +const sampleServer = createServer((req, res) => { + const name = path.basename(decodeURIComponent((req.url ?? '/').split('?')[0])); + if (!names.includes(name)) { + res.writeHead(404).end(); + return; + } + res.writeHead(200, { 'Content-Type': 'image/jpeg', 'Content-Length': jpg.length, 'Access-Control-Allow-Origin': '*' }); + res.end(jpg); +}); +await new Promise((resolve) => sampleServer.listen(SAMPLE_PORT, '127.0.0.1', resolve)); + +const app = process.env.BASE_URL + ? null + : spawn('npx', ['vite', 'preview', '--port', String(APP_PORT), '--strictPort', '--host', '127.0.0.1'], { + cwd: FRONTEND, + stdio: 'ignore', + detached: true, + }); +const stopApp = () => { + if (!app?.pid) return; + try { + process.kill(-app.pid, 'SIGTERM'); + } catch {} +}; +const base = process.env.BASE_URL ?? `http://127.0.0.1:${APP_PORT}`; +let up = !app; +for (let i = 0; i < 60 && !up; i++) { + up = await fetch(`${base}/library`).then((r) => r.ok).catch(() => false); + if (!up) await new Promise((r) => setTimeout(r, 500)); +} +if (!up) { + stopApp(); + sampleServer.close(); + console.log(`FAIL no preview on ${base}`); + process.exit(1); +} + +const browser = await playwright.chromium.launch({ + executablePath: process.env.CHROME || undefined, + args: ['--no-sandbox', '--enable-unsafe-swiftshader'], +}); +const context = await browser.newContext({ viewport: { width: 1500, height: 950 } }); + +await context.route('**/api/**', (route) => { + const url = route.request().url(); + const json = (body) => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(body) }); + if (url.includes('/auth/me')) return json({ user: { id: 1, email: 'check@example.com', name: 'check', verified: true, pro: true, admin: false } }); + if (url.includes('/photos')) return json({ photos: [] }); + if (url.includes('/place')) return json({ place: 'Hanoi' }); + if (url.includes('/ratings')) return json({ ratings: [] }); + if (url.includes('/highlights')) return json({ highlights: { day: [], week: [], ever: [] } }); + if (url.includes('/recipes')) return json({ recipes: [] }); + return json({}); +}); + +await context.addInitScript( + ({ origin, names, root }) => { + const fileHandle = (name) => { + const handle = { kind: 'file', name, __fake: true }; + Object.defineProperty(handle, 'getFile', { + value: async () => new File([await (await fetch(`${origin}/${name}`)).arrayBuffer()], name, { type: 'image/jpeg', lastModified: Date.UTC(2024, 3, 2, 12) }), + }); + return handle; + }; + const valuesOf = (node) => + [...node.files.map(fileHandle)][Symbol.iterator](); + const dirHandle = (label, node) => { + const handle = { kind: 'directory', name: label, __fake: true }; + Object.defineProperties(handle, { + values: { value: () => valuesOf(node) }, + queryPermission: { value: async () => 'granted' }, + requestPermission: { value: async () => 'granted' }, + getDirectoryHandle: { value: async () => { throw new Error('no folder'); } }, + getFileHandle: { + value: async (name) => { + if (!node.files.includes(name)) throw new Error(`no frame ${name}`); + return fileHandle(name); + }, + }, + }); + return handle; + }; + const rootNode = { files: names }; + window.showDirectoryPicker = async () => dirHandle(root, rootNode); + const revive = (row) => { + if (row?.handle?.__fake && !Object.getOwnPropertyDescriptor(row.handle, 'getFile')) { + row.handle = row.handle.kind === 'directory' ? dirHandle(root, rootNode) : fileHandle(row.handle.name); + } + return row; + }; + for (const method of ['get', 'getAll']) { + const original = IDBObjectStore.prototype[method]; + IDBObjectStore.prototype[method] = function (...args) { + const request = original.apply(this, args); + const descriptor = Object.getOwnPropertyDescriptor(IDBRequest.prototype, 'result'); + Object.defineProperty(request, 'result', { + configurable: true, + get() { + const value = descriptor.get.call(this); + return Array.isArray(value) ? value.map(revive) : revive(value); + }, + }); + return request; + }; + } + }, + { origin: `http://127.0.0.1:${SAMPLE_PORT}`, names, root: ROOT } +); + +const page = await context.newPage(); +const errors = []; +page.on('pageerror', (e) => errors.push(String(e.message).slice(0, 200))); +if (process.env.LOG) page.on('console', (m) => console.log('PAGE:', m.text().slice(0, 300))); + +const rows = () => + page.evaluate( + (db) => + new Promise((resolve) => { + const open = indexedDB.open(db); + open.onsuccess = () => { + const tx = open.result.transaction('photos', 'readonly'); + const req = tx.objectStore('photos').getAll(); + req.onsuccess = () => { + const photos = req.result.map((r) => ({ id: r.id, folder: r.folder, dir: r.dir, name: r.name })); + const t = open.result.transaction('thumbs', 'readonly'); + const tr = t.objectStore('thumbs').getAll(); + tr.onsuccess = () => resolve({ photos, thumbs: tr.result.map((x) => x.id) }); + }; + }; + open.onerror = () => resolve({ photos: [], thumbs: [] }); + }), + 'recipescam-library' + ); + +await page.goto(`${base}/library`); +// The catalogue the reader is left with: a drive of frames, each filed twice. +await page.evaluate( + async ({ names, root }) => { + const DB = 'recipescam-library'; + const STORES = { folders: 'name', photos: 'id', thumbs: 'id', edits: 'photoId', dirs: 'name' }; + const open = (version) => + new Promise((resolve, reject) => { + const req = version ? indexedDB.open(DB, version) : indexedDB.open(DB); + req.onupgradeneeded = () => { + for (const [store, keyPath] of Object.entries(STORES)) { + if (!req.result.objectStoreNames.contains(store)) req.result.createObjectStore(store, { keyPath }); + } + }; + req.onsuccess = () => resolve(req.result); + req.onerror = () => reject(req.error); + }); + let db = await open(); + if (!db.objectStoreNames.contains('thumbs')) { + const at = db.version; + db.close(); + db = await open(at + 1); + } + const handle = await window.showDirectoryPicker(); + const thumb = new Blob([new Uint8Array([1, 2, 3])], { type: 'image/jpeg' }); + await new Promise((resolve, reject) => { + const tx = db.transaction(['folders', 'photos', 'thumbs'], 'readwrite'); + tx.objectStore('folders').put({ name: root, label: 'D:', handle }); + for (const name of names) { + // The row the older build filed, and the row this build filed beside it + // when an UPDATE read the frame as new: one frame, two rows, two tiles. + for (const id of [`${root}/${name}`, `/${name}`]) { + tx.objectStore('photos').put({ + id, + folder: id.startsWith('/') ? '' : root, + name, + taken: Date.UTC(2024, 3, 2, 12), + size: 1, + addedAt: Date.now(), + }); + tx.objectStore('thumbs').put({ id, thumb }); + } + } + tx.oncomplete = () => resolve(); + tx.onerror = () => reject(tx.error); + }); + }, + { names, root: ROOT } +); + +await page.reload(); +await page.waitForSelector('[data-key^="lib-node-"]', { timeout: 60_000 }); +const before = await rows(); +step('the doubled catalogue is read', before.photos.length === names.length * 2, `${before.photos.length} rows: ${before.photos.map((p) => p.id).join(' | ')}`); + +// UPDATE on the drive row. +await page.locator('[data-key^="lib-node-"]').first().click({ button: 'right' }); +await page.waitForSelector('[data-key^="lib-update-"]', { timeout: 20_000 }); +await page.click('[data-key^="lib-update-"]'); +await page.waitForTimeout(8000); + +const after = await rows(); +step('UPDATE adds nothing', after.photos.length === names.length, `${after.photos.length} rows: ${after.photos.map((p) => p.id).join(' | ')}`); +step('one thumbnail per frame', after.thumbs.length === names.length, `${after.thumbs.length} thumbs: ${after.thumbs.join(' | ')}`); +step( + 'the doubled rows are dropped for the id this build computes', + after.photos.every((p) => p.id === `/${p.name}`) && after.thumbs.every((id) => id.startsWith('/')), + `rows: ${after.photos.map((p) => p.id).join(' | ')}` +); +step('no page error', errors.length === 0, errors.join(' | ')); + +await browser.close(); +stopApp(); +sampleServer.close(); +console.log(fails ? `${fails} FAIL` : 'all ok'); +process.exit(fails ? 1 : 0); diff --git a/docker/frontend/src/engine/library.ts b/docker/frontend/src/engine/library.ts index 93240dd..a397fdb 100644 --- a/docker/frontend/src/engine/library.ts +++ b/docker/frontend/src/engine/library.ts @@ -650,17 +650,42 @@ export async function removeFolder(name: string): Promise { // or a rotation belongs to a frame that is there. // // `from` is the folder the reading was pointed at, '' for the whole roll; `met` -// is every frame it found, spelled relative to the roll. -export async function prunePhotos(folder: string, from: string, met: Set): Promise { +// is every frame it found, spelled relative to the roll. `missing` says whether +// this reading has a word on the frames it did not meet — a reading that took up +// a position, or one that met a folder it could not open, has none and only +// clears what is said below about the frames it did meet. +export async function prunePhotos(folder: string, from: string, met: Set, missing = true): Promise { try { const root = normPath(folder); const under = (from ? `${root}/${normPath(from)}` : root).toLowerCase(); - const rows = (await listPhotos(folder)).filter((p) => { + // One frame, one row. A row this reading did not meet is a frame gone from the + // disk and goes with it — but a second row for a frame it did meet is the same + // frame filed under an id this build no longer computes, and the reader who + // asked for the disk is looking at one frame and not two: the first of the two + // rows stands, the rest go with their tiles and their marks. + const rows: LibraryPhoto[] = []; + const kept = new Map(); + for (const p of await listPhotos(folder)) { const rel = (p.dir ? `${normPath(p.dir)}/${p.name}` : p.name).toLowerCase(); const key = `${root.toLowerCase()}/${rel}`; - if (from && key !== under && !key.startsWith(`${under}/`)) return false; - return !met.has(rel); - }); + if (from && key !== under && !key.startsWith(`${under}/`)) continue; + if (!met.has(rel)) { + if (missing) rows.push(p); + continue; + } + const at = kept.get(key); + if (!at) { + kept.set(key, p); + continue; + } + // The row filed under the path this build computes is the one that stands; + // the other is the same frame under a spelling nothing reads, and it is + // this reading that has the frame in hand — so it goes, with its tile and + // its marks. + const stale = at.id.toLowerCase() === key ? p : at; + if (stale === at) kept.set(key, p); + rows.push(stale); + } if (!rows.length) return; const gone = new Set(rows.map((p) => p.id)); if (cached) cached = cached.filter((p) => !gone.has(p.id)); @@ -1188,9 +1213,22 @@ export async function scanFolder( if (!handle) throw new Error('no-folder-handle'); const rawKnown = await listPhotos(folder.name); const known = new Map(); + // The frame is its path, and the path is what two spellings of it have in + // common: a catalogue written by a build that filed a drive root's frames under + // `\/2017\a.RAF` — the raw name, before `photoId` folded the separators — holds + // this frame already, and a reading that looked it up by the id it computes + // would find nothing and file a second row and a second tile beside the first. + const knownPath = new Map(); + const pathIn = (p: LibraryPhoto) => + `${normPath(p.folder)}${p.dir ? `/${normPath(p.dir)}` : ''}/${p.name}`.toLowerCase(); for (const p of rawKnown) { known.set(p.id, p); known.set(p.id.toLowerCase(), p); + const path = pathIn(p); + // The row filed under the id this build computes is the one the frame is + // written back to; the other spelling is the same frame and stands only + // where it is the only row there is. + if (!knownPath.has(path) || p.id.toLowerCase() === path) knownPath.set(path, p); } // A reading kept to one folder keeps no position of its own. The position file // belongs to the reading that walks the whole roll, and a fraction of the tree @@ -1474,7 +1512,13 @@ export async function scanFolder( } const file = await withTimeout(handle.getFile(), READ_MS); const id = photoId(folder.name, rel); - const seen = known.get(id); + // The row the catalogue already holds for this frame, found by the id this + // reading computes and, failing that, by the path both spellings share — and + // the frame is written back under the id it is filed under: one frame is one + // row, one tile and one recipe, whatever a build before this one spelled it. + const held = known.get(id) ?? knownPath.get(`${normPath(folder.name)}/${normPath(rel)}`.toLowerCase()); + const filed = held?.id ?? id; + const seen = held; if (seen && seen.size === file.size && seen.mtime === file.lastModified) return; // Where the frame was taken, off the few hundred kilobytes of its head that // its own EXIF lives in — where the 32MB a camera's RAW used to be read for @@ -1495,7 +1539,7 @@ export async function scanFolder( const relNorm = rel.replace(/\\/g, '/'); const cut = relNorm.lastIndexOf('/'); const row: LibraryPhoto = { - id, + id: filed, folder: normPath(folder.name), dir: cut < 0 ? '' : normPath(relNorm.slice(0, cut)), name: handle.name, @@ -1661,8 +1705,11 @@ export async function scanFolder( // walked are folders this reading never listed — and one that met a folder it // could not open are all readings with no word on the frames they did not // reach, and what they did not reach is not theirs to drop. See `prunePhotos`. - if (!stop && !failed.size && (bounded || !resumed)) { - await prunePhotos(folder.name, bounded ? from : '', countedDiscovered); + // The frame met twice under one path is the other half of that and is dropped + // whatever the reading is: a row is said about here by the frame it is, and + // that frame this reading has just had in hand. + if (!stop) { + await prunePhotos(folder.name, bounded ? from : '', countedDiscovered, !failed.size && (bounded || !resumed)); } // What the walk found, kept where the catalogue is kept, so the column has it // on the next visit whether the reading ran to its end or was cut off. A @@ -2171,9 +2218,24 @@ async function writeThumb(id: string, blob: Blob, tell: boolean): Promise if (tell) for (const listener of thumbListeners) listener(id, blob); try { const db = await openDb(); - if (!db.objectStoreNames.contains(THUMBS)) return; - const tx = db.transaction(THUMBS, 'readwrite'); - tx.objectStore(THUMBS).put({ id, thumb: blob }); + if (!db.objectStoreNames.contains(THUMBS) || !db.objectStoreNames.contains(PHOTOS)) return; + // A tile belongs to a frame the catalogue holds: a frame the wall asked for + // the tile of is one it drew off a row — and a reading that runs behind the + // wall is a reading that may drop that row while the tile is being made. The + // tile would then be kept for a row nothing has, a store with one more tile + // in it than there are frames, which is `makeThumbs` reading the folder as + // short of a tile it will never be drawn out of. The read of the row and the + // write of the tile are one transaction, so the reading cannot step between + // them. + await new Promise((resolve, reject) => { + const tx = db.transaction([PHOTOS, THUMBS], 'readwrite'); + const req = tx.objectStore(PHOTOS).get(id); + req.onsuccess = () => { + if (req.result) tx.objectStore(THUMBS).put({ id, thumb: blob }); + }; + tx.oncomplete = () => resolve(); + tx.onerror = () => reject(tx.error); + }); } catch { // The tile is in hand either way; where it is not kept it is made again. } @@ -2466,38 +2528,56 @@ export async function readPhotos( // one folder's rows are the run of keys between the folder and its own name: // the rows the index handed back, asked of the store, a chunk at a time. const root = folder === undefined ? null : normPath(folder); - const at = (last: string | null): IDBKeyRange => - root === null - ? IDBKeyRange.lowerBound(last ?? '', last !== null) - : IDBKeyRange.bound(last ?? `${root}/`, `${root}/\uffff`, last !== null, false); + // A folder whose name carries a separator is a folder with two spellings: the + // picker hands a drive root `\` where `normPath` writes `''`, and the build + // that filed its frames before `photoId` folded the separators filed them all + // under the raw one — `\/2017\a.RAF` where this build writes `/2017/a.RAF`. + // A row in the other spelling is a row nothing reads: the reading of the disk + // meets the same frame again under the id it computes, finds nothing, and + // files a second row and a second tile beside the row it cannot see — and the + // repair that would drop it reads the folder the same way, so the two stay. + // The raw range is read with the computed one, and its rows come back under + // the ids they are filed under, for `scanFolder` to keep their frame from + // being filed twice and `prunePhotos` to drop against the spelling that + // stands. See both. + const raw = folder !== undefined && folder !== root ? folder : null; const rows: LibraryPhoto[] = []; - let last: string | null = null; let emittedFirst = false; - for (;;) { - const chunk = await new Promise((resolve, reject) => { - const req = db.transaction(PHOTOS, 'readonly').objectStore(PHOTOS).getAll(at(last), READ_CHUNK); - req.onsuccess = () => resolve(req.result as LibraryPhoto[]); - req.onerror = () => reject(req.error); - }); - if (!chunk.length) break; - last = chunk[chunk.length - 1].id; - for (const row of chunk) { - // Neither is read again: the file a frame is opened from is the folder's - // business — see `fileHandleFor` — and the tile is the store's own. - delete (row as { handle?: unknown }).handle; - delete (row as { thumb?: unknown }).thumb; - } - rows.push(...chunk); + const sweep = async (base: string): Promise => { + let last: string | null = null; + for (;;) { + const chunk = await new Promise((resolve, reject) => { + const range = + root === null + ? IDBKeyRange.lowerBound(last ?? '', last !== null) + : IDBKeyRange.bound(last ?? `${base}/`, `${base}/\uffff`, last !== null, false); + const req = db.transaction(PHOTOS, 'readonly').objectStore(PHOTOS).getAll(range, READ_CHUNK); + req.onsuccess = () => resolve(req.result as LibraryPhoto[]); + req.onerror = () => reject(req.error); + }); + if (!chunk.length) break; + last = chunk[chunk.length - 1].id; + for (const row of chunk) { + // Neither is read again: the file a frame is opened from is the folder's + // business — see `fileHandleFor` — and the tile is the store's own. + delete (row as { handle?: unknown }).handle; + delete (row as { thumb?: unknown }).thumb; + } + rows.push(...chunk); - if (onBatch && !emittedFirst && rows.length >= 400) { - emittedFirst = true; - const initial = [...rows].sort((a, b) => b.taken - a.taken); - onBatch(initial); - } + if (onBatch && !emittedFirst && rows.length >= 400) { + emittedFirst = true; + const initial = [...rows].sort((a, b) => b.taken - a.taken); + onBatch(initial); + } - if (chunk.length < READ_CHUNK) break; - } + if (chunk.length < READ_CHUNK) break; + } + }; + + await sweep(root ?? ''); + if (raw) await sweep(raw); rows.sort((a, b) => b.taken - a.taken); if (!folder) { cached = rows;