fix(library): read a frame's date with one set of options
exifr keeps a page-long cache of the options object it is handed, keyed by that object itself, so a fresh literal per frame pinned a frame's worth of tag sets (about thirty Sets) for as long as the tab lived: 3601 frames left ~16MB of heap behind, 9601 left ~50MB, and a roll of a hundred thousand frames is what a tab cannot hold. One options object per call site now, which is what that cache is for. Measured on the deployed bundle: 3601 frames rest at 8MB, not 23MB. scripts/ram-check.mjs reads a roll of many folders, queues folders behind the reading in front of them and reports the heap left after a forced collection, which is how those numbers were taken.
This commit is contained in:
@@ -0,0 +1,338 @@
|
||||
// What a reading costs the page, measured rather than argued: a roll of many
|
||||
// folders, a strip drawing it as it lands, and the heap sampled with a forced
|
||||
// collection between samples, so what is left is what somebody still holds.
|
||||
//
|
||||
// npm run build && node scripts/ram-check.mjs
|
||||
// APP_PORT=4950 SAMPLE_PORT=4951 SAMPLE_DIR=<sample folder> \
|
||||
// PLAYWRIGHT_CORE=<playwright-core/index.mjs> CHROME=<chrome> \
|
||||
// node scripts/ram-check.mjs
|
||||
import { spawn } from 'node:child_process';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const FRONTEND = fileURLToPath(new URL('..', import.meta.url));
|
||||
const SAMPLES = process.env.SAMPLE_DIR ?? path.resolve(FRONTEND, '..', 'sample');
|
||||
const APP_PORT = Number(process.env.APP_PORT ?? 4950);
|
||||
const SAMPLE_PORT = Number(process.env.SAMPLE_PORT ?? 4951);
|
||||
const JPG = 'P1010256.JPG';
|
||||
const RAW = 'P1010256.RW2';
|
||||
// One small JPEG served under every name: a hundred copies of an eight-megabyte
|
||||
// frame is eight hundred megabytes to prove a point about rows.
|
||||
const SMALL = 'Main.jpg';
|
||||
const SUBDIRS = Number(process.env.SUBDIRS ?? 12);
|
||||
const PER_DIR = Number(process.env.PER_DIR ?? 300);
|
||||
const QUEUE_AT = Number(process.env.QUEUE_AT ?? 4);
|
||||
const QUEUE_N = Number(process.env.QUEUE_N ?? 4);
|
||||
const BUDGET_MS = Number(process.env.BUDGET_MS ?? 90_000);
|
||||
const SLOW = Number(process.env.SLOW ?? 12);
|
||||
|
||||
const playwright = await import(process.env.PLAYWRIGHT_CORE ?? 'playwright-core').catch(() => null);
|
||||
if (!playwright) {
|
||||
console.log('SKIP playwright-core is not installed');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// The roll: a picked folder of `SUBDIRS` folders, each of `PER_DIR` frames.
|
||||
let n = 0;
|
||||
const rolls = {};
|
||||
const dirs = {};
|
||||
for (let d = 0; d < SUBDIRS; d++) {
|
||||
const names = [];
|
||||
for (let f = 0; f < PER_DIR; f++) names.push(`B${String(n++).padStart(6, '0')}.JPG`);
|
||||
dirs[`D${String(d).padStart(2, '0')}`] = { files: names, dirs: {} };
|
||||
}
|
||||
rolls.RamRoll = { files: [`B${String(n++).padStart(6, '0')}.JPG`], dirs };
|
||||
|
||||
const mobile = Number(process.env.MOBILE ?? 0);
|
||||
// BASE_URL runs the same reading against a stack already up (the deployed one),
|
||||
// so what is measured is the page the reader actually opens.
|
||||
const app = process.env.BASE_URL
|
||||
? null
|
||||
: spawn('npx', ['vite', 'preview', '--port', String(APP_PORT), '--strictPort', '--host', '127.0.0.1'], {
|
||||
cwd: FRONTEND,
|
||||
stdio: 'ignore',
|
||||
});
|
||||
const base = process.env.BASE_URL ?? `http://127.0.0.1:${APP_PORT}`;
|
||||
let up = false;
|
||||
for (let i = 0; i < 60 && !up; i++) {
|
||||
up = await fetch(`${base}/library`).then((r) => r.ok).catch(() => false);
|
||||
if (!up) await new Promise((r) => setTimeout(r, 500));
|
||||
}
|
||||
if (!up) {
|
||||
app?.kill('SIGTERM');
|
||||
console.log(`FAIL no preview on ${base} — run \`npm run build\` first`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const args = ['--no-sandbox', '--enable-unsafe-swiftshader'];
|
||||
if (mobile) args.push(`--js-flags=--max-old-space-size=${mobile}`);
|
||||
const browser = await playwright.chromium.launch({
|
||||
executablePath: process.env.CHROME || undefined,
|
||||
args,
|
||||
});
|
||||
const context = await browser.newContext();
|
||||
await context.route(`http://127.0.0.1:${SAMPLE_PORT}/**`, async (route) => {
|
||||
const name = path.basename(decodeURIComponent(new URL(route.request().url()).pathname));
|
||||
const file = /^B\d+\.JPG$/.test(name) ? SMALL : /\.jpe?g$/i.test(name) ? JPG : /\.rw2$/i.test(name) ? RAW : null;
|
||||
if (!file) return route.fulfill({ status: 404, body: '' });
|
||||
return route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/octet-stream',
|
||||
headers: { 'Access-Control-Allow-Origin': '*' },
|
||||
body: await readFile(path.join(SAMPLES, file)),
|
||||
});
|
||||
});
|
||||
await context.route('**/api/**', (route) => {
|
||||
const url = route.request().url();
|
||||
const json = (body) => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(body) });
|
||||
if (url.includes('/auth/me')) return json({ user: { id: 1, email: 'check@example.com', name: 'check' } });
|
||||
if (url.includes('/photos')) return json({ photos: [] });
|
||||
if (url.includes('/recipes')) return json({ recipes: [] });
|
||||
return route.fulfill({ status: 404, contentType: 'application/json', body: JSON.stringify({ error: 'no' }) });
|
||||
});
|
||||
|
||||
await context.addInitScript(
|
||||
({ origin, folder, rolls, small }) => {
|
||||
const bytes = new Map();
|
||||
const load = async (name) => {
|
||||
const file = /^B\d+\.JPG$/.test(name) ? small : name;
|
||||
if (!bytes.has(file)) bytes.set(file, fetch(`${origin}/${file}`).then((r) => r.arrayBuffer()));
|
||||
return bytes.get(file);
|
||||
};
|
||||
const fileHandle = (name) => {
|
||||
const handle = { kind: 'file', name, __fake: true };
|
||||
Object.defineProperty(handle, 'getFile', {
|
||||
value: async () => {
|
||||
// A read of a frame is nearly all waiting on a real disk, and a
|
||||
// reading that is over in a blink is one the heap cannot be watched
|
||||
// against. `check.slow` is that wait, in milliseconds.
|
||||
const slow = Number(localStorage.getItem('check.slow') ?? 0);
|
||||
if (slow) await new Promise((r) => setTimeout(r, slow));
|
||||
return new File([await load(name)], name, { type: 'application/octet-stream', lastModified: Date.UTC(2026, 0, 1) });
|
||||
},
|
||||
});
|
||||
return handle;
|
||||
};
|
||||
const dirHandle = (label, node) => {
|
||||
const handle = { kind: 'directory', name: label, __fake: true };
|
||||
Object.defineProperties(handle, {
|
||||
values: {
|
||||
value: () => {
|
||||
if (node.fail) throw new Error(`${label} will not open`);
|
||||
return [
|
||||
...(node.files ?? []).map(fileHandle),
|
||||
...Object.entries(node.dirs ?? {}).map(([name, kid]) => dirHandle(name, kid)),
|
||||
][Symbol.iterator]();
|
||||
},
|
||||
},
|
||||
queryPermission: { value: async () => 'granted' },
|
||||
requestPermission: { value: async () => 'granted' },
|
||||
getDirectoryHandle: {
|
||||
value: async (name) => {
|
||||
const kid = node.dirs?.[name];
|
||||
if (!kid) throw new Error(`no folder named ${name}`);
|
||||
return dirHandle(name, kid);
|
||||
},
|
||||
},
|
||||
getFileHandle: {
|
||||
value: async (name) => {
|
||||
if (!(node.files ?? []).includes(name)) throw new Error(`no frame named ${name}`);
|
||||
return fileHandle(name);
|
||||
},
|
||||
},
|
||||
});
|
||||
return handle;
|
||||
};
|
||||
const root = () => dirHandle(folder, { files: rolls[folder].files, dirs: rolls[folder].dirs });
|
||||
window.showDirectoryPicker = async () => root();
|
||||
const revive = (row) => {
|
||||
if (row?.handle?.__fake && !Object.getOwnPropertyDescriptor(row.handle, 'getFile')) {
|
||||
row.handle = row.handle.kind === 'directory' ? root() : fileHandle(row.handle.name);
|
||||
}
|
||||
return row;
|
||||
};
|
||||
for (const method of ['get', 'getAll']) {
|
||||
const original = IDBObjectStore.prototype[method];
|
||||
IDBObjectStore.prototype[method] = function (...args) {
|
||||
const request = original.apply(this, args);
|
||||
const descriptor = Object.getOwnPropertyDescriptor(IDBRequest.prototype, 'result');
|
||||
Object.defineProperty(request, 'result', {
|
||||
configurable: true,
|
||||
get() {
|
||||
const value = descriptor.get.call(this);
|
||||
return Array.isArray(value) ? value.map(revive) : revive(value);
|
||||
},
|
||||
});
|
||||
return request;
|
||||
};
|
||||
}
|
||||
},
|
||||
{ origin: `http://127.0.0.1:${SAMPLE_PORT}`, folder: 'RamRoll', small: SMALL, rolls }
|
||||
);
|
||||
|
||||
const page = await context.newPage();
|
||||
const cdp = await context.newCDPSession(page);
|
||||
await cdp.send('Performance.enable');
|
||||
const sample = async () => {
|
||||
await cdp.send('HeapProfiler.collectGarbage');
|
||||
const { metrics } = await cdp.send('Performance.getMetrics');
|
||||
const m = Object.fromEntries(metrics.map((x) => [x.name, x.value]));
|
||||
const dom = await page.evaluate(() => document.querySelectorAll('*').length);
|
||||
return { heap: m.JSHeapUsedSize, nodes: m.Nodes, dom };
|
||||
};
|
||||
const line = () => page.$eval('[data-key="lib-progress"]', (e) => (e.textContent ?? '').trim()).catch(() => '');
|
||||
const stored = () =>
|
||||
page
|
||||
.evaluate(async () => {
|
||||
const db = await new Promise((resolve, reject) => {
|
||||
const req = indexedDB.open('recipescam-library');
|
||||
req.onsuccess = () => resolve(req.result);
|
||||
req.onerror = () => reject(req.error);
|
||||
});
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = db.transaction('photos', 'readonly').objectStore('photos').count();
|
||||
req.onsuccess = () => resolve(req.result);
|
||||
req.onerror = () => reject(req.error);
|
||||
});
|
||||
})
|
||||
.catch(() => -1);
|
||||
|
||||
await page.goto(`${base}/library`);
|
||||
await page.evaluate((s) => localStorage.setItem('check.slow', String(s)), SLOW);
|
||||
await page.click('[data-key="lib-add"]');
|
||||
await page.waitForSelector('[data-key="lib-node-RamRoll"]', { timeout: 60_000 });
|
||||
if (Number(process.env.PROFILE ?? 0)) await cdp.send('HeapProfiler.startSampling', { samplingInterval: 4096 });
|
||||
|
||||
const series = [];
|
||||
const t0 = Date.now();
|
||||
let queued = false;
|
||||
while (Date.now() - t0 < BUDGET_MS) {
|
||||
const s = await sample();
|
||||
s.t = Math.round((Date.now() - t0) / 100) / 10;
|
||||
s.line = await line();
|
||||
series.push(s);
|
||||
// The reader right-clicks a few folders under the roll: each is a reading
|
||||
// queued behind the one running, which is what the report is about.
|
||||
if (!queued && series.length >= QUEUE_AT) {
|
||||
queued = true;
|
||||
const keys = await page.evaluate(() => [...document.querySelectorAll('[data-key^="lib-node-RamRoll/D"]')].map((e) => e.dataset.key));
|
||||
for (const key of keys.slice(0, QUEUE_N)) {
|
||||
await page.click(`[data-key="${key}"]`, { button: 'right' }).catch(() => {});
|
||||
await page.click('[data-key="lib-update-RamRoll"]', { timeout: 5000 }).catch(() => {});
|
||||
await page.waitForTimeout(150);
|
||||
}
|
||||
console.log(`queued ${Math.min(QUEUE_N, keys.length)} folders`);
|
||||
}
|
||||
await page.waitForTimeout(1000);
|
||||
}
|
||||
const rest = await sample();
|
||||
const first = series[0] ?? { heap: 0 };
|
||||
const peak = series.reduce((a, b) => (b.heap > a.heap ? b : a), first);
|
||||
const last = series[series.length - 1] ?? first;
|
||||
console.log(`samples ${series.length} over ${last.t}s rows ${await stored()}`);
|
||||
console.log(`heap first ${Math.round(first.heap / 1e6)}MB peak ${Math.round(peak.heap / 1e6)}MB@${peak.t}s last ${Math.round(last.heap / 1e6)}MB resting ${Math.round(rest.heap / 1e6)}MB`);
|
||||
console.log(`nodes first ${first.nodes} dom ${first.dom} -> nodes ${last.nodes} dom ${last.dom}`);
|
||||
if (Number(process.env.SNAPSHOT ?? 0)) {
|
||||
const chunks = [];
|
||||
cdp.on('HeapProfiler.addHeapSnapshotChunk', (e) => chunks.push(e.chunk));
|
||||
await cdp.send('HeapProfiler.enable');
|
||||
await cdp.send('HeapProfiler.takeHeapSnapshot', { reportProgress: false });
|
||||
const snap = JSON.parse(chunks.join(''));
|
||||
const fields = snap.snapshot.meta.node_fields;
|
||||
const types = snap.snapshot.meta.node_types[fields.indexOf('type')];
|
||||
const stride = fields.length;
|
||||
const at = (i, f) => snap.nodes[i * stride + fields.indexOf(f)];
|
||||
const byKey = new Map();
|
||||
for (let i = 0; i < snap.nodes.length / stride; i++) {
|
||||
const key = `${types[at(i, 'type')]} ${snap.strings[at(i, 'name')]}`;
|
||||
const cur = byKey.get(key) ?? { n: 0, bytes: 0 };
|
||||
cur.n++;
|
||||
cur.bytes += at(i, 'self_size');
|
||||
byKey.set(key, cur);
|
||||
}
|
||||
console.log('--- who is alive (self_size) ---');
|
||||
for (const [key, v] of [...byKey.entries()].sort((a, b) => b[1].bytes - a[1].bytes).slice(0, 30)) {
|
||||
console.log(` ${String(Math.round(v.bytes / 1024)).padStart(8)}KB ${String(v.n).padStart(7)}x ${key}`);
|
||||
}
|
||||
if (Number(process.env.RETAIN ?? 0)) {
|
||||
// Who keeps the named object alive: walk the edges into the first node of a
|
||||
// given name and print the path back to a root the way the panel does.
|
||||
const nodeCount = snap.nodes.length / stride;
|
||||
const eStride = snap.snapshot.meta.edge_fields.length;
|
||||
let maxTo = 0;
|
||||
for (let i = 0; i < snap.edges.length / eStride; i++) maxTo = Math.max(maxTo, snap.edges[i * eStride + 2]);
|
||||
const scale = maxTo >= nodeCount ? eStride : 1;
|
||||
console.log(` nodes ${nodeCount} edges ${snap.edges.length / eStride} maxTo ${maxTo} -> to_node scale ${scale}`);
|
||||
const firstEdge = new Int32Array(nodeCount + 1);
|
||||
{
|
||||
let run = 0;
|
||||
for (let i = 0; i < nodeCount; i++) {
|
||||
firstEdge[i] = run;
|
||||
run += at(i, 'edge_count');
|
||||
}
|
||||
firstEdge[nodeCount] = run;
|
||||
}
|
||||
const want = process.env.RETAIN;
|
||||
const typesOf = (i) => types[at(i, 'type')];
|
||||
const nameOf = (i) => snap.strings[at(i, 'name')];
|
||||
const targets = [];
|
||||
for (let i = 0; i < nodeCount && targets.length < 5; i++) if (typesOf(i) === 'object' && nameOf(i) === want) targets.push(i);
|
||||
const parent = new Map();
|
||||
const queue = [0];
|
||||
let head = 0;
|
||||
const seen = new Uint8Array(nodeCount);
|
||||
seen[0] = 1;
|
||||
const wantSet = new Set(targets);
|
||||
let hit = -1;
|
||||
while (head < queue.length) {
|
||||
const n0 = queue[head++];
|
||||
if (wantSet.has(n0)) {
|
||||
hit = n0;
|
||||
break;
|
||||
}
|
||||
for (let e = firstEdge[n0]; e < firstEdge[n0 + 1]; e++) {
|
||||
const child = snap.edges[e * eStride + 2] / scale;
|
||||
if (!seen[child]) {
|
||||
seen[child] = 1;
|
||||
parent.set(child, [n0, snap.strings[snap.edges[e * eStride]]]);
|
||||
queue.push(child);
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(`--- retained path to "${want}" (${targets.length} found) ---`);
|
||||
if (hit >= 0) {
|
||||
const path = [];
|
||||
let cur = hit;
|
||||
while (cur !== undefined && cur !== 0) {
|
||||
const p = parent.get(cur);
|
||||
if (!p) break;
|
||||
path.push(`${nameOf(cur)}(${typesOf(cur)}) -[${p[1]}]-`);
|
||||
cur = p[0];
|
||||
}
|
||||
console.log(` ${path.reverse().join(' ')}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (Number(process.env.PROFILE ?? 0)) {
|
||||
const { profile } = await cdp.send('HeapProfiler.stopSampling');
|
||||
const rows = [];
|
||||
const walk = (node, stack) => {
|
||||
const f = node.callFrame ?? {};
|
||||
const here = stack.concat([`${f.functionName || '(anonymous)'}@${(f.url ?? '').split('/').pop()}:${(f.lineNumber ?? 0) + 1}`]);
|
||||
if (node.selfSize) rows.push({ name: f.functionName || '(anonymous)', url: (f.url ?? '').split('/').pop(), size: node.selfSize, stack: here.slice(-6).join(' <- ') });
|
||||
for (const kid of node.children ?? []) walk(kid, here);
|
||||
};
|
||||
walk(profile.head, []);
|
||||
console.log('--- who allocates (selfSize, leaf) ---');
|
||||
for (const r of [...rows].sort((a, b) => b.size - a.size).slice(0, 25)) console.log(` ${String(Math.round(r.size / 1024)).padStart(8)}KB ${r.name} ${r.url}`);
|
||||
const byStack = new Map();
|
||||
for (const r of rows) byStack.set(r.stack, (byStack.get(r.stack) ?? 0) + r.size);
|
||||
console.log('--- where it comes from (stack, selfSize summed) ---');
|
||||
for (const [stack, size] of [...byStack.entries()].sort((a, b) => b[1] - a[1]).slice(0, 25)) {
|
||||
console.log(` ${String(Math.round(size / 1024)).padStart(8)}KB ${stack}`);
|
||||
}
|
||||
}
|
||||
for (const s of series) console.log(` ${String(s.t).padStart(6)}s ${String(Math.round(s.heap / 1e6)).padStart(5)}MB nodes ${String(s.nodes).padStart(7)} dom ${String(s.dom).padStart(7)} ${s.line}`);
|
||||
await browser.close();
|
||||
app?.kill('SIGTERM');
|
||||
@@ -151,11 +151,31 @@ async function shrinkOrKeep(bytes: Uint8Array, maxDim: number, maxBytes: number)
|
||||
// hands it back as a Date in ours — which is what a camera prints on a stamp, and
|
||||
// what someone opening a 2019 photo today expects to read under the coordinates.
|
||||
// Null when the file carries no date at all; the caller falls back to its clock.
|
||||
// `exifr` keeps a private cache of the parsed options it is handed, keyed by the
|
||||
// options object itself, and every entry in it lives as long as the page. A
|
||||
// fresh literal per frame is therefore a frame's worth of tag sets held forever:
|
||||
// reading a roll of a hundred thousand frames is what fills the tab. So the
|
||||
// options are written once here and the same object is handed over every time —
|
||||
// which is also what the cache is for.
|
||||
const DATE_OPTIONS = { pick: ['DateTimeOriginal', 'CreateDate', 'ModifyDate'] };
|
||||
const SPEC_OPTIONS = {
|
||||
pick: [
|
||||
'Make',
|
||||
'Model',
|
||||
'ISO',
|
||||
'ISOSpeedRatings',
|
||||
'FocalLength',
|
||||
'FNumber',
|
||||
'ExposureTime',
|
||||
'ExifImageWidth',
|
||||
'ExifImageHeight',
|
||||
],
|
||||
};
|
||||
const ISO_OPTIONS = { pick: ['ISO', 'ISOSpeedRatings'] };
|
||||
|
||||
export async function readCapturedAt(bytes: Uint8Array): Promise<number | null> {
|
||||
try {
|
||||
const tags = (await exifr.parse(bytes, {
|
||||
pick: ['DateTimeOriginal', 'CreateDate', 'ModifyDate'],
|
||||
})) as Record<string, unknown> | undefined;
|
||||
const tags = (await exifr.parse(bytes, DATE_OPTIONS)) as Record<string, unknown> | undefined;
|
||||
for (const tag of ['DateTimeOriginal', 'CreateDate', 'ModifyDate']) {
|
||||
const value = tags?.[tag];
|
||||
if (value instanceof Date && Number.isFinite(value.getTime())) return value.getTime();
|
||||
@@ -187,9 +207,7 @@ export interface ShotSpecs {
|
||||
|
||||
export async function readSpecs(bytes: Uint8Array): Promise<ShotSpecs | null> {
|
||||
try {
|
||||
const tags = (await exifr.parse(bytes, {
|
||||
pick: ['Make', 'Model', 'ISO', 'ISOSpeedRatings', 'FocalLength', 'FNumber', 'ExposureTime', 'ExifImageWidth', 'ExifImageHeight'],
|
||||
})) as Record<string, unknown> | undefined;
|
||||
const tags = (await exifr.parse(bytes, SPEC_OPTIONS)) as Record<string, unknown> | undefined;
|
||||
if (!tags) return null;
|
||||
const num = (v: unknown): number | null => {
|
||||
const n = Number(Array.isArray(v) ? v[0] : v);
|
||||
@@ -263,7 +281,7 @@ export async function readGps(bytes: Uint8Array): Promise<GPSInfo | null> {
|
||||
// the caller then prints "AUTO" instead of inventing a number.
|
||||
export async function readIso(bytes: Uint8Array): Promise<number | null> {
|
||||
try {
|
||||
const tags = (await exifr.parse(bytes, { pick: ['ISO', 'ISOSpeedRatings'] })) as
|
||||
const tags = (await exifr.parse(bytes, ISO_OPTIONS)) as
|
||||
| { ISO?: unknown; ISOSpeedRatings?: unknown }
|
||||
| undefined;
|
||||
const raw = tags?.ISO ?? tags?.ISOSpeedRatings;
|
||||
|
||||
Reference in New Issue
Block a user