3dtours/RecipesCam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: the camera's own colour, a reading that keeps its place, and cloud links in Settings

Browse Source 
- RAW opens at the colour the camera wrote into it: the develop carries the
  difference against the file's own preview (previewResidual) instead of the
  per-channel cubic fit, dE00 6.04 -> 0.54; previewMatch.ts keeps only the grid.
- Immich reading: keyset position kept across visits, one reading per node,
  a page in flight retried, timeline buckets for the years on offer.
- The wall draws a server frame at the address the proxy serves it (no blob
  minted for it), and the 80-thumbnail pre-cache on open is gone.
- Settings lists the cloud links, flags two that answer at the same address,
  and forgets one.
- Custom frames imported from a file (engine/customFrame.ts, IndexedDB).
- Upload an edited frame from Studio to Immich Cloud.
This commit is contained in:
3dtours
2026-10-12 07:59:24 +07:00
parent 807c271b6b
commit 97fd181cfa
25 changed files with 2402 additions and 616 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/frontend/src/engine/customFrame.ts
+253
View File
@@ -0,0 +1,253 @@
// Custom imported transparent photo frame engine and storage
export interface CustomFrameHole {
x: number; // 0..1 fraction of frame width
y: number; // 0..1 fraction of frame height
w: number; // 0..1 fraction of frame width
h: number; // 0..1 fraction of frame height
}
export interface CustomFrameData {
id: string;
name: string;
bytes: Uint8Array;
dataUrl: string;
width: number;
height: number;
hole: CustomFrameHole;
scale: number; // default 1.0 (cover/fill hole)
panX: number; // offset in frame pixels
panY: number; // offset in frame pixels
}
const DB_NAME = 'recipescam-studio';
const STORE = 'custom_frames';
const FRAME_KEY = 'active_custom_frame';
function openDb(): Promise<IDBDatabase> {
return new Promise((resolve, reject) => {
const req = indexedDB.open(DB_NAME, 2);
req.onupgradeneeded = () => {
const db = req.result;
if (!db.objectStoreNames.contains('session')) {
db.createObjectStore('session');
}
if (!db.objectStoreNames.contains(STORE)) {
db.createObjectStore(STORE);
}
};
req.onsuccess = () => resolve(req.result);
req.onerror = () => reject(req.error);
});
}
export async function saveCustomFrame(frame: CustomFrameData): Promise<void> {
try {
const db = await openDb();
await new Promise<void>((resolve, reject) => {
const tx = db.transaction(STORE, 'readwrite');
tx.objectStore(STORE).put(frame, FRAME_KEY);
tx.oncomplete = () => resolve();
tx.onerror = () => reject(tx.error);
});
db.close();
} catch (err) {
console.warn('Failed to save custom frame to IndexedDB', err);
}
}
export async function loadCustomFrame(): Promise<CustomFrameData | null> {
try {
const db = await openDb();
const row = await new Promise<CustomFrameData | undefined>((resolve, reject) => {
const tx = db.transaction(STORE, 'readonly');
const req = tx.objectStore(STORE).get(FRAME_KEY);
req.onsuccess = () => resolve(req.result);
req.onerror = () => reject(req.error);
});
db.close();
if (row && row.bytes instanceof Uint8Array && row.hole) {
return row;
}
return null;
} catch {
return null;
}
}
export async function clearCustomFrame(): Promise<void> {
try {
const db = await openDb();
await new Promise<void>((resolve, reject) => {
const tx = db.transaction(STORE, 'readwrite');
tx.objectStore(STORE).delete(FRAME_KEY);
tx.oncomplete = () => resolve();
tx.onerror = () => reject(tx.error);
});
db.close();
} catch {}
}
/**
* Analyzes an imported frame file (PNG, WebP, SVG, etc.),
* decodes it, and detects the transparent opening/hole via alpha channel analysis.
*/
export async function analyzeCustomFrame(file: File): Promise<CustomFrameData> {
const bytes = new Uint8Array(await file.arrayBuffer());
const blob = new Blob([bytes], { type: file.type || 'image/png' });
const dataUrl = await new Promise<string>((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => resolve(reader.result as string);
reader.onerror = reject;
reader.readAsDataURL(blob);
});
const img = new Image();
await new Promise<void>((resolve, reject) => {
img.onload = () => resolve();
img.onerror = () => reject(new Error('Cannot decode frame image'));
img.src = dataUrl;
});
const w = img.naturalWidth || img.width;
const h = img.naturalHeight || img.height;
if (!(w > 0 && h > 0)) {
throw new Error('Invalid image dimensions');
}
// Downscale slightly for fast and accurate pixel analysis (max 600px)
const maxDim = 600;
const sampleScale = Math.min(1, maxDim / Math.max(w, h));
const sw = Math.max(10, Math.round(w * sampleScale));
const sh = Math.max(10, Math.round(h * sampleScale));
const canvas = document.createElement('canvas');
canvas.width = sw;
canvas.height = sh;
const ctx = canvas.getContext('2d', { willReadFrequently: true });
if (!ctx) {
throw new Error('Canvas 2D context not available');
}
ctx.drawImage(img, 0, 0, sw, sh);
const imgData = ctx.getImageData(0, 0, sw, sh);
const data = imgData.data;
// Transparent threshold: alpha < 70 (out of 255)
const isAlphaLow = (x: number, y: number) => {
return data[(y * sw + x) * 4 + 3] < 70;
};
// 1. Flood-fill from outer edges to distinguish outer transparent padding from the inner transparent hole
const visited = new Uint8Array(sw * sh);
const queue: number[] = [];
for (let x = 0; x < sw; x++) {
if (isAlphaLow(x, 0)) {
visited[0 * sw + x] = 1;
queue.push(x, 0);
}
if (isAlphaLow(x, sh - 1)) {
visited[(sh - 1) * sw + x] = 1;
queue.push(x, sh - 1);
}
}
for (let y = 0; y < sh; y++) {
if (isAlphaLow(0, y) && !visited[y * sw + 0]) {
visited[y * sw + 0] = 1;
queue.push(0, y);
}
if (isAlphaLow(sw - 1, y) && !visited[y * sw + (sw - 1)]) {
visited[y * sw + (sw - 1)] = 1;
queue.push(sw - 1, y);
}
}
let head = 0;
while (head < queue.length) {
const qx = queue[head++];
const qy = queue[head++];
const neighbors = [
[qx + 1, qy],
[qx - 1, qy],
[qx, qy + 1],
[qx, qy - 1],
];
for (const [nx, ny] of neighbors) {
if (nx >= 0 && nx < sw && ny >= 0 && ny < sh) {
const idx = ny * sw + nx;
if (!visited[idx] && isAlphaLow(nx, ny)) {
visited[idx] = 1;
queue.push(nx, ny);
}
}
}
}
// 2. Find bounding box of inner transparent pixels (not reached by flood-fill from border)
let minX = sw;
let minY = sh;
let maxX = -1;
let maxY = -1;
let innerTransparentCount = 0;
for (let y = 0; y < sh; y++) {
for (let x = 0; x < sw; x++) {
const idx = y * sw + x;
if (isAlphaLow(x, y) && !visited[idx]) {
innerTransparentCount++;
if (x < minX) minX = x;
if (x > maxX) maxX = x;
if (y < minY) minY = y;
if (y > maxY) maxY = y;
}
}
}
// 3. Fallback: if no inner transparent pixel was isolated, search for any transparent pixels
if (innerTransparentCount < 20 || minX > maxX) {
minX = sw;
minY = sh;
maxX = -1;
maxY = -1;
let anyCount = 0;
for (let y = 0; y < sh; y++) {
for (let x = 0; x < sw; x++) {
if (isAlphaLow(x, y)) {
anyCount++;
if (x < minX) minX = x;
if (x > maxX) maxX = x;
if (y < minY) minY = y;
if (y > maxY) maxY = y;
}
}
}
innerTransparentCount = anyCount;
}
// 4. Compute normalized hole geometry
let hole: CustomFrameHole = { x: 0.1, y: 0.1, w: 0.8, h: 0.8 };
if (innerTransparentCount > 20 && minX <= maxX && minY <= maxY) {
const holeW = Math.max(0.05, (maxX - minX + 1) / sw);
const holeH = Math.max(0.05, (maxY - minY + 1) / sh);
hole = {
x: minX / sw,
y: minY / sh,
w: holeW,
h: holeH,
};
}
return {
id: `custom-${Date.now()}`,
name: file.name,
bytes,
dataUrl,
width: w,
height: h,
hole,
scale: 1.0,
panX: 0,
panY: 0,
};
}
docker/frontend/src/engine/exportEngine.ts
+76 -2
View File
@@ -100,6 +100,13 @@ export interface RenderOptions {
// quarter turn and the fine angle (they ride on top of both).
photoFlipH?: boolean;
photoFlipV?: boolean;
customFrame?: {
bytes: Uint8Array;
hole: { x: number; y: number; w: number; h: number };
scale?: number;
panX?: number;
panY?: number;
} | null;
// The strip the FILE carries: the amber `#TAG` over the photo and a caption
// band appended below it, mirroring the landing page's `.lp-shot` +
// `.lp-frame-meta`. Left off the live preview, which must not show it.
@@ -770,8 +777,8 @@ function loadWallframe(assets: RenderAssets) {
// is a transparent window a couple of percent of paper wide around it.
const isOldFilm = (id: FrameId) => id === 'old-film' || id === 'old-film-portrait';
// A frame that carries its own opening: the aspect chip has nothing to say to
// it (both walls, both films).
const hasOwnAspect = (id: FrameId) => isWallframe(id) || isOldFilm(id);
// it (both walls, both films, custom frames).
const hasOwnAspect = (id: FrameId) => isWallframe(id) || isOldFilm(id) || id === 'custom';
function loadOldFilm(assets: RenderAssets) {
if (oldFilmImage) return oldFilmImage;
@@ -1521,6 +1528,73 @@ export async function renderPhoto(input: RenderInput): Promise<RenderResult | nu
stampCanvas = cardCanvas;
stampW = outW;
stampH = outH;
} else if (frameId === 'custom' && options?.customFrame) {
flush(surface);
const photoImg = own(surface.makeImageSnapshot());
cardSource = photoImg;
const pw = photoImg.width();
const ph = photoImg.height();
const customImg = own(Skia.Image.MakeImageFromEncoded(options.customFrame.bytes));
if (!customImg) {
console.error('Failed to decode custom frame');
return null;
}
const fw = customImg.width();
const fh = customImg.height();
const cardSurface = createSurface(fw, fh);
if (!cardSurface) return null;
own(cardSurface);
const cardCanvas = cardSurface.getCanvas();
const plainPaint = own(Skia.Paint());
const hole = options.customFrame.hole;
const holeX = hole.x * fw;
const holeY = hole.y * fh;
const holeW = hole.w * fw;
const holeH = hole.h * fh;
const holeCenterX = holeX + holeW / 2;
const holeCenterY = holeY + holeH / 2;
// Base cover scale to fill the transparent hole
const baseScale = Math.max(holeW / pw, holeH / ph);
const zoom = options.customFrame.scale ?? 1.0;
const finalScale = baseScale * zoom;
const drawW = pw * finalScale;
const drawH = ph * finalScale;
const panX = options.customFrame.panX ?? 0;
const panY = options.customFrame.panY ?? 0;
const drawX = holeCenterX + panX - drawW / 2;
const drawY = holeCenterY + panY - drawH / 2;
// 1. Draw photo underneath
cardCanvas.drawImageRectOptions(
photoImg,
Skia.XYWHRect(0, 0, pw, ph),
Skia.XYWHRect(drawX, drawY, drawW, drawH),
Skia.FilterMode.Linear,
Skia.MipmapMode.None,
plainPaint
);
// 2. Draw custom frame on top so transparent area displays the photo
cardCanvas.drawImageRectOptions(
customImg,
Skia.XYWHRect(0, 0, fw, fh),
Skia.XYWHRect(0, 0, fw, fh),
Skia.FilterMode.Linear,
Skia.MipmapMode.None,
plainPaint
);
outputSurface = cardSurface;
stampCanvas = cardCanvas;
stampW = fw;
stampH = fh;
} else {
drawFrameOnCanvas(canvas, width, height, frameId);
}
docker/frontend/src/engine/immich.ts
+288 -59
View File
@@ -14,6 +14,7 @@ import {
putFolder,
putPhotoThumb,
putPhotos,
putPhotosFast,
remoteUrl,
saveWalkedDirs,
type LibraryDir,
@@ -25,6 +26,17 @@ import {
// there), so this is the size the sync loops at: one round trip per two hundred
// frames rather than one per frame.
const PAGE = 200;
// How many times one page of a server's frames is asked for again before the
// reading gives up on it. A page in flight is a call to a box the reader's own
// network may be between, and the second answer is usually the first answer.
const ASSET_RETRIES = 2;
// The nodes one reading is walking through. The screen asks for a sync when it
// opens and whenever the node it shows changes, and two readings of one node at
// the same time are two readings asking the server for the same pages and writing
// one place over the other: the second is turned away, and the first keeps the
// place for both.
const walking = new Set<string>();
// Tiles are drawn after the rows have landed rather than before: the column and
// the count are what the reader sees first, and a tile is the wall's own business.
const TILE_LANES = 3;
@@ -155,6 +167,37 @@ export async function forgetImmichServer(id: string): Promise<boolean> {
}
}
// Upload an edited photo to Immich Cloud
export async function uploadToImmich(
bytes: Uint8Array,
name: string,
serverId?: string,
albumId?: string
): Promise<{ ok: boolean; id?: string; error?: string }> {
try {
const params = new URLSearchParams();
if (serverId) params.set('server', serverId);
if (name) params.set('name', name);
if (albumId) params.set('album', albumId);
const res = await fetch(`/api/immich/upload?${params.toString()}`, {
method: 'POST',
headers: {
'content-type': 'image/jpeg',
},
body: bytes as BodyInit,
});
if (!res.ok) {
const err = await res.json().catch(() => ({}));
return { ok: false, error: (err as any)?.error || (err as any)?.details || `HTTP ${res.status}` };
}
const data = await res.json().catch(() => ({}));
return { ok: true, id: (data as any)?.id };
} catch (err: any) {
return { ok: false, error: err?.message || String(err) };
}
}
// The albums this key can see, asked for live: an album added on Immich has to be
// there the next time the dialog is opened, so nothing here is cached.
export async function listImmichAlbums(server: string): Promise<ImmichAlbum[]> {
@@ -180,17 +223,48 @@ export async function saveImmichAlbums(server: string, selected: string[]): Prom
// One page of a server's frames. `album` is an Immich id or null for everything
// the key can see — the album field's empty state.
async function listAssets(
export async function listAssets(
server: string,
album: string | null,
page: number,
search = ''
): Promise<{ items: ImmichAsset[]; hasMore: boolean }> {
search = '',
takenBefore?: string | null,
takenAfter?: string | null
): Promise<{ items: ImmichAsset[]; hasMore: boolean; ok: boolean }> {
const query = new URLSearchParams({ server, page: String(page), size: String(PAGE) });
if (album) query.set('album', album);
if (search) query.set('q', search);
const body = await get<{ items?: ImmichAsset[]; hasMore?: boolean }>(`/api/immich/assets?${query.toString()}`);
return { items: body?.items ?? [], hasMore: body?.hasMore === true };
if (takenBefore) query.set('takenBefore', takenBefore);
if (takenAfter) query.set('takenAfter', takenAfter);
// A page that could not be read is not a page with nothing on it. Both are one
// shape to a loop that counts items, and the loop that takes the one for the
// other reads the end of the roll at the first timeout and drops every row it
// had not met yet. `ok` is the word that says which of the two this was, and an
// unanswered page is asked for again here rather than in every caller.
for (let attempt = 0; ; attempt++) {
try {
const res = await fetch(`/api/immich/assets?${query.toString()}`);
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const body = await json<{ items?: ImmichAsset[]; hasMore?: boolean }>(res);
if (!body) throw new Error('unreadable answer');
return { items: body.items ?? [], hasMore: body.hasMore === true, ok: true };
} catch {
if (attempt >= ASSET_RETRIES) return { items: [], hasMore: false, ok: false };
await new Promise((resolve) => setTimeout(resolve, 250 * (attempt + 1)));
}
}
}
// Fetch timeline buckets (months/years) for an Immich server in one fast call
export async function fetchImmichBuckets(
server: string,
size: 'YEAR' | 'MONTH' = 'MONTH'
): Promise<{ timeBucket: string; count: number }[]> {
const query = new URLSearchParams({ server, size });
const body = await get<{ buckets?: { timeBucket: string; count: number }[] }>(
`/api/immich/buckets?${query.toString()}`
);
return Array.isArray(body?.buckets) ? body.buckets : [];
}
// What the tree paints, and the name an album's frames are filed under. An album
@@ -221,6 +295,11 @@ function frameName(name: string, id: string, taken: Set<string>): string {
export interface ImmichSyncResult {
total: number;
albums: number;
// False while the node is still being read — the answer above comes back at the
// foreground window, and the rest of it is read behind it. The screen says so
// rather than a number that would have the reader believe the node holds six
// hundred frames.
done: boolean;
// The album rows this read named, spelled the way the column spells a path. The
// screen puts these in the place of the node's old ones: an album the reader
// unticked is a row that has to go, and a record that is only ever added to
@@ -228,15 +307,20 @@ export interface ImmichSyncResult {
dirs: LibraryDir[];
}
export type ImmichSyncCallback = (batch: LibraryPhoto[], progress: { loaded: number; hasMore: boolean }) => void;
// Read a server's albums into the catalogue: the rows and the column first, so
// the count and the tree are right the moment this returns, and the tiles after,
// a lane at a time, so a reader who opens the node while it is still fetching
// sees frames rather than a blank wall.
//
// One read covers the whole node. There is no per-album read: the album list is
// one call, a page of frames is one call, and a column that says one album is
// fresh while the others are a week old is a column nobody can read a number off.
export async function syncImmich(folder: LibraryFolder): Promise<ImmichSyncResult> {
// Streams batches to IndexedDB and UI immediately (in < 200ms per batch).
// Resolves the foreground pass in 3 pages (up to 600 newest photos) so the UI
// unblocks instantly, and continues syncing historical photos in the background.
export async function syncImmich(
folder: LibraryFolder,
onBatch?: ImmichSyncCallback
): Promise<ImmichSyncResult> {
const server = folder.server ?? '';
const search = folder.kind === 'search' ? (folder.query ?? '').trim() : '';
const chosen = folder.albums ?? [];
@@ -245,20 +329,78 @@ export async function syncImmich(folder: LibraryFolder): Promise<ImmichSyncResul
const rows: LibraryPhoto[] = [];
const dirs: LibraryDir[] = [];
const counts: Record<string, number> = {};
// The frames this reading has already filed. The keyset read stops on a frame's
// own time and `takenBefore` is inclusive, so every page opens on the frame the
// page before it ended with: filed as it came, that frame is a row the wall
// draws twice. `frameName` answers the name a second *file* takes; the id is
// what says there is no second file.
const seen = new Set<string>();
// 1. ALWAYS read ALL photos of the Immich API (filed under root of folder: dir = '')
// This guarantees that all photos (timeline, camera roll, uploads) are fully loaded into the Library.
// Keyset pagination with takenBefore prevents slow SQL offsets and timeouts!
const rootTaken = new Set<string>();
let page = 1;
// Where the reading before this one stopped, when it did not reach the end. A
// node of a hundred thousand frames is not one visit's to read, and a place only
// ever held in memory is a place every reload began without: the whole of what
// the catalogue was given was then the newest three pages of it, once a visit,
// for good. The node's own record keeps the place instead — the frame it stopped
// on, and the page within that frame's own instant.
const resume = folder.cursor && !folder.cursor.done ? folder.cursor : null;
let lastTakenBefore: string | null = resume?.before ?? null;
let page = resume?.page ?? 1;
let pages = 0;
let rootCount = 0;
let complete = false;
let failed = false;
const FOREGROUND_PAGES = 3; // The newest three pages answer the call so the screen unblocks
let hasMoreRemaining = false;
// The place, written down where the visit after this one reads it. Every page
// rather than every minute: a page is two hundred frames, and a visit may end
// at any of them.
const remember = async (before: string | null, at: number, done: boolean) => {
try {
await putFolder({ ...folder, cursor: { before, page: at, done } });
} catch {}
};
// Where a page leaves the reading. The bound is a frame's own instant and the
// call that asks for it keeps the frame it stopped on — `takenBefore` is
// inclusive — so a page whose oldest frame is older than the bound has moved it,
// and the page after it starts at that very frame. A page that is one instant
// from end to end cannot move it at all: Immich keeps a frame's time to the
// millisecond, and a roll imported in one go is hundreds of frames at the same
// one, which is the whole of the oldest days of this library. Asked for with the
// bound alone, that page is the same page for ever — the reading never ends, and
// the frames under that instant are never read. So the rest of the instant is
// walked by offset, and the bound is left where it is until the pages under it
// run out.
// ponytail: that offset walk is over everything older than the bound, and a deep
// offset is a slow answer — an Immich that offers a cursor for it, or a page
// asked per instant out of the timeline buckets, is the way out of it if a roll
// of thousands at one instant ever sits in the middle of a library.
const step = (items: ImmichAsset[]): void => {
const at = items[items.length - 1]?.takenAt ?? null;
if (page === 1 && at && at !== lastTakenBefore) lastTakenBefore = at;
else page++;
};
for (;;) {
const { items, hasMore } = await listAssets(server, null, page, search);
const { items, hasMore, ok } = await listAssets(server, null, page, search, lastTakenBefore);
// A page that could not be read is not the end of the node, and only a
// reading that met the end of it may say what the node no longer holds.
if (!ok) {
failed = true;
break;
}
const batchRows: LibraryPhoto[] = [];
for (const item of items) {
if (!item.id || !item.name) continue;
if (!item.id || !item.name || seen.has(item.id)) continue;
seen.add(item.id);
const name = frameName(item.name, item.id, rootTaken);
rootTaken.add(item.name);
rootTaken.add(name);
rows.push({
const row: LibraryPhoto = {
id: [root, name].join('/'),
folder: folder.name,
dir: '',
@@ -268,17 +410,35 @@ export async function syncImmich(folder: LibraryFolder): Promise<ImmichSyncResul
taken: item.takenAt ? Date.parse(item.takenAt) || Date.now() : Date.now(),
size: item.size ?? 0,
addedAt: Date.now(),
});
};
rows.push(row);
batchRows.push(row);
rootCount++;
}
if (!hasMore || !items.length) break;
page++;
// Save batch to IndexedDB and push to UI immediately
if (batchRows.length) {
await putPhotosFast(batchRows);
onBatch?.(batchRows, { loaded: rootCount, hasMore });
}
pages++;
if (!hasMore || !items.length) {
complete = true;
break;
}
step(items);
await remember(lastTakenBefore, page, false);
// After foreground window, resolve foreground sync promptly so UI is responsive
if (pages >= FOREGROUND_PAGES) {
hasMoreRemaining = true;
break;
}
}
counts[root] = rootCount;
// 2. ALBUMS of Immich ONLY display in the ALBUM section if the user chooses specific albums (chosen.length > 0).
// When chosen is empty, dirs remains empty ([]) so no album subfolders clutter the ALBUM tree.
// When chosen has album IDs, only those specific chosen albums are fetched and populated into dirs.
if (!search && chosen.length > 0) {
const albums = await listImmichAlbums(server);
const labels = albumLabels(albums);
@@ -290,13 +450,21 @@ export async function syncImmich(folder: LibraryFolder): Promise<ImmichSyncResul
let albPage = 1;
let albCount = 0;
for (;;) {
const { items, hasMore } = await listAssets(server, album.id, albPage, search);
const { items, hasMore, ok } = await listAssets(server, album.id, albPage, search);
// An album that could not be read is one this reading has no word on: the
// rows it holds are left where they are, and nothing below is pruned
// against a reading that did not meet them.
if (!ok) {
failed = true;
break;
}
const albBatch: LibraryPhoto[] = [];
for (const item of items) {
if (!item.id || !item.name) continue;
const name = frameName(item.name, item.id, albumTaken);
albumTaken.add(item.name);
albumTaken.add(name);
rows.push({
const row: LibraryPhoto = {
id: [root, label, name].join('/'),
folder: folder.name,
dir: label,
@@ -306,9 +474,15 @@ export async function syncImmich(folder: LibraryFolder): Promise<ImmichSyncResul
taken: item.takenAt ? Date.parse(item.takenAt) || Date.now() : Date.now(),
size: item.size ?? 0,
addedAt: Date.now(),
});
};
rows.push(row);
albBatch.push(row);
albCount++;
}
if (albBatch.length) {
await putPhotosFast(albBatch);
onBatch?.(albBatch, { loaded: rootCount + albCount, hasMore });
}
if (!hasMore || !items.length) break;
albPage++;
}
@@ -317,50 +491,105 @@ export async function syncImmich(folder: LibraryFolder): Promise<ImmichSyncResul
}
}
await saveWalkedDirs(root, dirs, counts, true);
// A frame the server no longer lists is not a frame to keep drawing: the row,
// its tile and its marks go, exactly as a frame taken off the disk does. This
// read saw the whole node, so it is the one allowed to say it.
const met = new Set(rows.map((r) => (r.dir ? `${normPath(r.dir)}/${r.name}` : r.name).toLowerCase()));
await prunePhotos(root, '', met);
await putPhotos(rows);
// The record carries the reading's whole word or none of it: only a reading
// that met the end of the node may say what the node holds, because a record
// that says otherwise is the record the column reads its noughts off. A reading
// stopped at the foreground window, or cut short by a page that would not come
// back, folds its half into what was already there instead.
await saveWalkedDirs(root, dirs, counts, complete && !failed);
// Pre-cache thumbnails for initial display without stalling large libraries
const missing = new Set(await missingThumbs(rows.map((r) => r.id)));
const queue = rows.filter((r) => missing.has(r.id));
const initialBatch = queue.slice(0, 80);
const backgroundBatch = queue.slice(80);
// A row the reading did not meet is dropped only against a reading that met all
// of them — and only one that began at the top. A reading that carried on from
// where the last one stopped holds the newest pages in its own list and no older
// row in it at all, and pruned to that list it would take back every frame it
// was picking up.
// ponytail: a resumed reading therefore stops one prune short — a frame dropped
// on the server keeps its row until some visit reads the node from the top. Read
// `met` out of the catalogue rather than out of `rows` if that ever shows.
if (complete && !failed && !resume) {
const met = new Set(rows.map((r) => (r.dir ? `${normPath(r.dir)}/${r.name}` : r.name).toLowerCase()));
await prunePhotos(root, '', met);
}
await Promise.all(
Array.from({ length: Math.min(TILE_LANES, initialBatch.length) }, async () => {
while (initialBatch.length) {
const row = initialBatch.shift()!;
try {
const res = await fetch(remoteUrl('thumbnail', row));
if (res.ok) await putPhotoThumb(row.id, await res.blob());
} catch {
// One frame's tile is one frame's: the wall draws a blank cell and the
// reader may ask again, rather than a sync that stops halfway.
}
await breathe();
}
})
);
if (backgroundBatch.length) {
// The rest of the node, a page at a time, behind the answer above. Its place is
// the record's exactly as the foreground's is, so a visit that ends in the
// middle of it — a reload, a closed tab, a lid shut — costs the page in flight
// and nothing else.
if ((hasMoreRemaining || failed) && !walking.has(root)) {
walking.add(root);
void (async () => {
while (backgroundBatch.length) {
const row = backgroundBatch.shift()!;
try {
const res = await fetch(remoteUrl('thumbnail', row));
if (res.ok) await putPhotoThumb(row.id, await res.blob());
} catch {}
await breathe();
let bgPage = page;
let bgLastTakenBefore = lastTakenBefore;
let reachedEnd = false;
try {
for (;;) {
const { items, hasMore: bgMore, ok } = await listAssets(
server,
null,
bgPage,
search,
bgLastTakenBefore
);
// A page that could not be read leaves the place where the last page
// that could left it: the visit after this one asks for it again.
if (!ok) break;
if (!items.length) {
reachedEnd = true;
break;
}
const bgBatchRows: LibraryPhoto[] = [];
for (const item of items) {
if (!item.id || !item.name || seen.has(item.id)) continue;
seen.add(item.id);
const name = frameName(item.name, item.id, rootTaken);
rootTaken.add(item.name);
rootTaken.add(name);
bgBatchRows.push({
id: [root, name].join('/'),
folder: folder.name,
dir: '',
name,
server,
remoteId: item.id,
taken: item.takenAt ? Date.parse(item.takenAt) || Date.now() : Date.now(),
size: item.size ?? 0,
addedAt: Date.now(),
});
rootCount++;
}
if (bgBatchRows.length) {
await putPhotosFast(bgBatchRows);
onBatch?.(bgBatchRows, { loaded: rootCount, hasMore: bgMore });
}
if (!bgMore) {
reachedEnd = true;
break;
}
const at = items[items.length - 1]?.takenAt ?? null;
if (bgPage === 1 && at && at !== bgLastTakenBefore) bgLastTakenBefore = at;
else bgPage++;
await remember(bgLastTakenBefore, bgPage, false);
await breathe();
}
counts[root] = rootCount;
await saveWalkedDirs(root, dirs, counts, reachedEnd);
if (reachedEnd) {
// The node is read through: the place is not a place any more, and the
// reading that begins next begins at the top — which is where a frame
// added since would be.
await remember(null, 1, true);
if (!resume) {
const met = new Set(rows.map((r) => (r.dir ? `${normPath(r.dir)}/${r.name}` : r.name).toLowerCase()));
await prunePhotos(root, '', met);
}
}
} finally {
walking.delete(root);
}
})();
}
return { total: rootCount, albums: dirs.length, dirs };
return { total: rootCount, albums: dirs.length, dirs, done: complete && !failed };
}
// The name a server's node takes: the first one is `Immich`, and a second is
docker/frontend/src/engine/library.ts
+75 -22
View File
@@ -25,7 +25,7 @@ const PHOTOS = 'photos';
const THUMBS = 'thumbs';
const EDITS = 'edits';
const DIRS = 'dirs';
const DB_VERSION = 4;
const DB_VERSION = 5;
// The tile the catalogue keeps is the same tile the grid cell and the strip both
// paint, and there is one of them per frame of a roll that reaches six figures.
@@ -112,6 +112,17 @@ export interface LibraryFolder {
// the name is the store's key — but a label the reader may rename is no place to
// keep a query, and the sync asks with this.
query?: string;
// Where a server's reading stopped, on a node whose frames are an Immich
// server's. A node of a hundred thousand frames is not one visit's to read, and
// the keyset the reading pages by is a value the page cannot hold across a
// reload: without this, every visit read the newest three pages of the node
// again and nothing below them was ever read at all. `before` is the instant the
// next page starts from, `page` the page within that instant — Immich keeps a
// frame's time to the millisecond and a roll imported in one go is hundreds of
// frames at the same one, so one instant is several pages — and `done` says the
// whole node has been read, which is the only state in which a frame that is no
// longer there may be dropped from the catalogue.
cursor?: { before: string | null; page?: number; done: boolean };
}
export interface LibraryPhoto {
@@ -322,11 +333,14 @@ function openDb(): Promise<IDBDatabase> {
// to be made anyway.
if (db.objectStoreNames.contains(THUMBS)) db.deleteObjectStore(THUMBS);
if (!db.objectStoreNames.contains(FOLDERS)) db.createObjectStore(FOLDERS, { keyPath: 'name' });
let photosStore: IDBObjectStore;
if (!db.objectStoreNames.contains(PHOTOS)) {
const photos = db.createObjectStore(PHOTOS, { keyPath: 'id' });
photos.createIndex('folder', 'folder');
photos.createIndex('taken', 'taken');
photosStore = db.createObjectStore(PHOTOS, { keyPath: 'id' });
} else {
photosStore = req.transaction!.objectStore(PHOTOS);
}
if (!photosStore.indexNames.contains('folder')) photosStore.createIndex('folder', 'folder');
if (!photosStore.indexNames.contains('taken')) photosStore.createIndex('taken', 'taken');
if (!db.objectStoreNames.contains(THUMBS)) {
db.createObjectStore(THUMBS, { keyPath: 'id' });
}
@@ -2047,9 +2061,9 @@ function pump(): void {
let cached: LibraryPhoto[] | null = null;
export function cachedPhotos(): LibraryPhoto[] {
if (cached) return cached;
if (cached && cached.length > 0) return cached;
try {
const raw = sessionStorage.getItem('rc_cached_photos');
const raw = sessionStorage.getItem('rc_cached_photos') || localStorage.getItem('rc_cached_photos');
if (raw) {
const parsed = JSON.parse(raw);
if (Array.isArray(parsed) && parsed.length > 0) {
@@ -2061,6 +2075,26 @@ export function cachedPhotos(): LibraryPhoto[] {
return [];
}
export function saveCachedPhotos(photos: LibraryPhoto[]): void {
try {
const slim = photos.slice(0, 300).map((p) => ({
id: p.id,
folder: p.folder,
name: p.name,
taken: p.taken,
addedAt: p.addedAt,
dir: p.dir,
star: p.star,
server: p.server,
remoteId: p.remoteId,
rot: p.rot,
}));
const json = JSON.stringify(slim);
sessionStorage.setItem('rc_cached_photos', json);
localStorage.setItem('rc_cached_photos', json);
} catch {}
}
export function clearLibraryCache(): void {
cached = null;
dirHandles.clear();
@@ -2385,6 +2419,10 @@ export async function makeThumbs(
const dir = normPath(rel);
const rows = Array.isArray(folder) ? folder : (await readPhotos(normPath(folder.name))) ?? [];
const want = rows.filter((p) => {
// A server's frame has no tile to mint here: the wall draws the address the
// proxy serves it at (see `tile` in Library.tsx), so reading the bytes into
// the store is the same picture fetched twice and a copy kept for nothing.
if (p.remoteId) return false;
const pDir = normPath(p.dir ?? '');
if (pDir === dir) return true;
// `''` is the picked folder itself, so a row asked of the whole roll takes
@@ -2413,7 +2451,10 @@ export async function makeThumbs(
return made;
}
export async function readPhotos(folder?: string): Promise<LibraryPhoto[] | null> {
export async function readPhotos(
folder?: string,
onBatch?: (photos: LibraryPhoto[]) => void
): Promise<LibraryPhoto[] | null> {
try {
const db = await openDb();
// Every row is filed under `normPath(folder.name)`, and a drive root is the
@@ -2431,6 +2472,8 @@ export async function readPhotos(folder?: string): Promise<LibraryPhoto[] | null
: IDBKeyRange.bound(last ?? `${root}/`, `${root}/\uffff`, last !== null, false);
const rows: LibraryPhoto[] = [];
let last: string | null = null;
let emittedFirst = false;
for (;;) {
const chunk = await new Promise<LibraryPhoto[]>((resolve, reject) => {
const req = db.transaction(PHOTOS, 'readonly').objectStore(PHOTOS).getAll(at(last), READ_CHUNK);
@@ -2446,26 +2489,19 @@ export async function readPhotos(folder?: string): Promise<LibraryPhoto[] | null
delete (row as { thumb?: unknown }).thumb;
}
rows.push(...chunk);
if (onBatch && !emittedFirst && rows.length >= 400) {
emittedFirst = true;
const initial = [...rows].sort((a, b) => b.taken - a.taken);
onBatch(initial);
}
if (chunk.length < READ_CHUNK) break;
}
rows.sort((a, b) => b.taken - a.taken);
if (!folder) {
cached = rows;
try {
const slim = rows.slice(0, 300).map((p) => ({
id: p.id,
folder: p.folder,
name: p.name,
taken: p.taken,
addedAt: p.addedAt,
dir: p.dir,
star: p.star,
server: p.server,
remoteId: p.remoteId,
rot: p.rot,
}));
sessionStorage.setItem('rc_cached_photos', JSON.stringify(slim));
} catch {}
saveCachedPhotos(rows);
}
return rows;
} catch {
@@ -2473,6 +2509,23 @@ export async function readPhotos(folder?: string): Promise<LibraryPhoto[] | null
}
}
export async function putPhotosFast(rows: LibraryPhoto[]): Promise<void> {
if (!rows.length) return;
const db = await openDb();
await new Promise<void>((resolve, reject) => {
const tx = db.transaction(PHOTOS, 'readwrite');
const store = tx.objectStore(PHOTOS);
for (const row of rows) {
const r = { ...row };
delete (r as { handle?: unknown }).handle;
delete (r as { thumb?: unknown }).thumb;
store.put(r);
}
tx.oncomplete = () => resolve();
tx.onerror = () => reject(tx.error);
});
}
export async function listPhotos(folder?: string): Promise<LibraryPhoto[]> {
return (await readPhotos(folder)) ?? [];
}
docker/frontend/src/engine/previewMatch.ts
+14 -123
View File
@@ -1,127 +1,18 @@
// The colour the camera itself wrote into the file.
// The grid the frame's own rendering is read on.
//
// LibRaw hands back the sensor's own data and nothing about how the body chose to
// render it: the white balance it locked, its picture style, its tone curve. The
// file carries the answer anyway — the preview saved inside it, which is the
// frame the photographer saw and the one a desktop viewer shows. So the develop
// is matched to that preview: develop a grid of the frame, take the same grid out
// of the embedded preview, and fit the curve between them — one cubic a channel,
// on the sRGB-encoded value the develop leaves.
// This file held the fit that used to relate a develop to the preview the camera
// wrote into the file: a grid out of each, a cubic a channel between them, and a
// 3x3 least squared ahead of that curve. None of it is the camera's rendering —
// a picture style is a colour transform with a curve in it — and the best a
// per-channel cubic could reach on the Fujifilm RAF this was reported on was
// dE00 6.04 against the file's own preview.
//
// A 3x3 was fitted ahead of that curve for a long time, least squared from one
// grid to the other, and it is what put the cast the studio was reported with:
// a develop carries its luminance and its colour in the same three numbers, the
// luminance is two orders of magnitude the larger of the two, and a least squares
// that is dominated by it pays for the luminance with the colour — it holds the
// channel means and the bright/dark axis while it collapses the axis the eye
// reads as colour. The camera's own picture style is a curve, not a matrix.
// What replaced it is not a fit: the develop carries the difference itself, at
// the preview's own resolution, and adds it back on the value it left (see
// previewResidual in rawDevelop.ts). dE00 0.54 on that frame, 0.00 on every body
// that writes a full-size preview.
//
// Measured on the Olympus ORF this was reported on, standard deviation of R-G
// over the frame (the file's own preview 8.3, the camera's own JPEG 12.3, and an
// olive frame is one where this has gone to nothing):
// develop as the sensor left it 11.6
// the curve alone, no 3x3 11.9
// the curve under the fitted 3x3 (what used to ship) 4.6
// and on the frame's own means, curve alone against the preview: luminance 156.6
// against 156.8, R/G 1.023 against 1.014, B/G 0.853 against 0.843.
//
// The fit is on the sRGB-encoded values, exposure included: the reference is the
// file's own frame, so matching its brightness is part of matching its colour.
// Fitted in that encoded space, not in linear light, because a least squares in
// linear light weighs the highlights and comes back with the midtones wrong —
// the same frames scored dE00 6.4 fitted encoded against 9.2 fitted linear. Any
// body, any picture style, and no fitted table to go stale — a preview is the one
// profile every RAW already carries.
// The grid the fit reads, and the size of the preview map handed to it.
// What is left here is the grid both the balance read and the develop's own maps
// are taken on, which is still the size the preview is reduced to when something
// only needs a statistic of the frame.
export const MATCH_GRID = 64;
// A block at or past this has nothing left to say about a rendering.
const CLIP = 0.97;
const FLOOR = 0.03;
// What the shader draws when the frame has no curve the fit can trust.
export const FLAT_TONE = new Float32Array([0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0]);
// The curve a channel needs to reach the camera's, least squared: the develop's
// own value for that channel against the block the camera wrote there.
//
// The cubic is pinned at BOTH black (x=0 -> y=0) and white (x=1 -> y=1).
// Pinned at both ends, a blown pixel (1,1,1) stays pure white, and a deep shadow
// pixel (0,0,0) stays pure black without shifting to purple or magenta noise.
function basis(i: number, x: number): number {
const x3 = x * x * x;
return i === 0 ? x - x3 : x * x - x3;
}
function solve2x2(A: number[][], d: number[]): number[] {
const det = A[0][0] * A[1][1] - A[0][1] * A[1][0] + 1e-9;
const a = (d[0] * A[1][1] - d[1] * A[0][1]) / det;
const b = (A[0][0] * d[1] - A[1][0] * d[0]) / det;
return [a, b];
}
// The curve itself, in the shader's own order: r a b c d, then g, then b.
function fitTone(blocks: { x: number[]; t: number[] }[]): Float32Array {
const out = new Float32Array(12);
for (let c = 0; c < 3; c++) {
const A = [[0, 0], [0, 0]];
const d = [0, 0];
for (const { x, t } of blocks) {
const v = Math.min(1, Math.max(0, x[c]));
const b = [basis(0, v), basis(1, v)];
for (let p = 0; p < 2; p++) {
for (let k = 0; k < 2; k++) A[p][k] += b[p] * b[k];
d[p] += b[p] * (t[c] - v * v * v);
}
}
const [a, b] = solve2x2(A, d);
out.set([0, a, b, 1 - a - b], c * 4);
}
return out;
}
// A curve that comes back folded, or wandered far from the value it was handed,
// is a fit off blocks that fought each other, not a rendering: the caller opens
// the frame as the sensor left it instead. The floor sits below the zero a curve
// is allowed to reach — the develop clamps there anyway — and the slope is only
// asked not to turn back.
function curveIsSane(tone: Float32Array): boolean {
for (const v of tone) if (!Number.isFinite(v) || Math.abs(v) > 4) return false;
for (let c = 0; c < 3; c++) {
const [a, b, cc, dd] = [tone[c * 4], tone[c * 4 + 1], tone[c * 4 + 2], tone[c * 4 + 3]];
let prev = a;
for (let i = 1; i <= 16; i++) {
const x = i / 16;
const y = a + x * (b + x * (cc + x * dd));
if (y < prev - 0.12 || y < -0.15 || y > 1.15) return false;
prev = y;
}
}
return true;
}
// `dev` and `ref` are n x n RGBA maps of the same frame, one byte per channel,
// both sRGB-encoded by the same resampler. Null when the two cannot be related —
// too few blocks carry colour — and the caller develops the frame as the sensor
// left it.
export function toneMatch(dev: Uint8Array, ref: Uint8Array, n = MATCH_GRID): Float32Array | null {
const blocks: { x: number[]; t: number[] }[] = [];
let used = 0;
for (let i = 0; i < n * n; i++) {
const x = [dev[i * 4] / 255, dev[i * 4 + 1] / 255, dev[i * 4 + 2] / 255];
const t = [ref[i * 4] / 255, ref[i * 4 + 1] / 255, ref[i * 4 + 2] / 255];
// A block the camera clipped has no colour left to copy, one at the floor is
// only the pedestal, and one the develop clipped has lost the ratio between
// its channels — none of them say anything about the rendering.
if (t[0] > CLIP && t[1] > CLIP && t[2] > CLIP) continue;
if (x[0] > CLIP && x[1] > CLIP && x[2] > CLIP) continue;
if (Math.max(t[0], t[1], t[2]) < FLOOR) continue;
used++;
blocks.push({ x, t });
}
// A fit off a handful of blocks is a fit off noise.
if (used < (n * n) / 8) return null;
const tone = fitTone(blocks);
return curveIsSane(tone) ? tone : null;
}
docker/frontend/src/engine/rawDevelop.ts
+221 -84
View File
@@ -1,32 +1,36 @@
// RAW → JPEG on the client, so a camera's own file opens in the studio without
// a DNG converter in the middle (see native_raw_processing_opfs_architecture.md).
//
// A RAW opens at the colour the camera chose for it. LibRaw is kept out of white
// balance and tone (below) and knows nothing of the body's picture style, so the
// develop is fitted to the one rendering the file does carry with it: the preview
// the camera wrote inside it, which is the frame the photographer saw and the one
// a desktop viewer shows. Develop the sensor, take a block grid out of each on
// the encoded values, fit the cubic a channel between them, and develop the
// sensor through that curve (see previewMatch.ts).
// A RAW opens at the colour the camera chose for it, and at the sensor's own
// resolution. LibRaw is kept out of white balance and tone (below) and knows
// nothing of the body's picture style; the one rendering the file does carry with
// it is the preview the camera wrote inside it — the frame the photographer saw
// and the one a desktop viewer shows.
//
// The curve is the whole fit. A 3x3 was least squared ahead of it for a long
// time and is what put the olive cast on the Olympus frame this was reported on:
// a develop keeps its luminance and its colour in the same three numbers and the
// luminance is two orders of magnitude the larger, so a least squares dominated
// by it holds the channel means and buys them with the colour axis. The camera's
// picture style is a curve, not a matrix.
// The develop is carried by the difference between that preview and this develop's
// own rendering of the same frame. The difference is kept at the preview's own
// resolution, one byte a channel with 0.5 for a pixel the two already agree on,
// and added back on the value the develop leaves: the frame opens at the camera's
// colour and at the sensor's resolution, and the develop is only ever asked for
// the detail the preview never had. Nothing is fitted — a body is matched against
// its own preview the moment it opens, whatever its picture style is, and there is
// no table to go stale (see previewResidual below).
//
// Measured on that frame, standard deviation of R-G over it (the file's own
// preview 8.3, the camera's own JPEG 12.3, an olive frame being one where this
// has gone to nothing): 11.6 as the develop left it, 11.9 through the curve
// alone, 4.6 through the curve under the fitted 3x3. What no per-pixel transform
// reaches either way is the camera's own sharpening.
// It replaced a per-channel cubic fitted between a grid of each, and a 3x3 least
// squared ahead of that curve before it, neither of which is the camera's
// rendering: a picture style is a colour transform with a curve in it. Measured
// against the file's own preview, mean dE00: 6.04 for the cubic on the Fujifilm
// RAF this was reported on, 0.54 for the residual on that frame and 0.00 on every
// body whose preview is the size of its sensor (a Nikon NEF, a DNG), where the two
// frames agree block for block and the residual is nothing.
//
// Developing the sensor is also what keeps the highlights: 2.3% of the frame at
// pure white against 3.3% in the preview itself, since the 8-bit preview threw
// the headroom away and the develop still has it. The preview is therefore the
// reference and the fallback, not the frame: it is what a file with no usable
// preview (some DNG), or with sensor data that will not decode, opens as.
// What the residual gives up is the headroom: it lands the frame on the preview's
// own white, so the two stops the sensor holds above white are not kept (the cubic
// kept 2.3% of the frame at pure white against 3.3% in the preview itself). A
// highlight recovery that needs them has to come from the develop, not from here.
//
// The preview is also the fallback, not the frame: it is what a file with no
// usable preview (some DNG), or with sensor data that will not decode, opens as.
//
// The develop: LibRaw demosaics in its own worker; what comes back is linear camera data,
// which this file turns into the sRGB the rest of the pipeline expects. The
@@ -53,7 +57,7 @@
import LibRaw from 'libraw-wasm';
import { sensorWhite } from './sensorWhite';
import { f32ToF16 } from './halfFloat';
import { toneMatch, FLAT_TONE, MATCH_GRID } from './previewMatch';
import { MATCH_GRID } from './previewMatch';
import { Skia, initSkia } from './skiaShim';
import { decodeTiff } from './tiffDecode';
@@ -95,15 +99,8 @@ uniform float4 m0; // camera -> sRGB, the first three columns of rgb_cam
uniform float4 m1;
uniform float4 m2;
uniform float4 crop; // (y offset of this band, 0, 0, 0)
uniform float4 t0; // the file's own rendering, one cubic a channel on the
uniform float4 t1; // encoded value: r a b c d per float4 (flat when there is
uniform float4 t2; // no preview to fit to)
// The camera's rendering is a shape, and this is that shape, read on the value
// the develop left.
float tone(float4 w, float x) {
return clamp(w.x + x * (w.y + x * (w.z + x * w.w)), 0.0, 1.0);
}
uniform shader res; // the file's own rendering, as a residual (see below)
uniform float2 rscl; // frame pixels -> residual pixels
float3 encode(float3 x) {
x = clamp(x, 0.0, 1.0);
@@ -166,24 +163,33 @@ half4 main(float2 pos) {
rgb *= (0.7 + over / (1.0 + over * 3.3333)) / mx;
}
float3 e = encode(rgb);
// The file's own colour: the curve, in float, on the encoded value it was fitted
// on — and the last step the frame leaves through. Not the 8-bit colour filter
// this used to be painted through: the curve carries an exposure (the preview is
// the reference, so matching its brightness is part of matching its colour), and
// it is what carries the body's own tone — a 3x3 can only scale, and the one that
// used to sit here bought the channel means by collapsing the colour axis it was
// fitted against (see previewMatch.ts). No rolloff: white is one of the values
// the curve is fitted on, and it is pinned there, so a blown pixel still lands on
// white while the frame's own highlights stay where the knee above left them.
float3 o = float3(tone(t0, e.r), tone(t1, e.g), tone(t2, e.b));
// ...and the neutral the gate above drew is what the curve undoes: one cubic a
// channel, fitted on a grid that has no block left to fit where the frame ran out
// (the blown blocks are excluded, see previewMatch), so at the plateau the three
// curves agree only at 1.0 and part company either side of it — measured on the
// ORF, a 255,255,255 sky came back 254,255,255 and 253,255,254, red under green
// across a quarter of the frame. The gate is the develop's own statement that the
// pixel had no colour of its own, so it is re-read here, on the value that leaves.
return half4(half3(mix(o, float3(max(max(o.r, o.g), o.b)), blown)), 1.0);
// The neutral the gate above drew, kept on the way out.
float3 o = mix(e, float3(max(max(e.r, e.g), e.b)), blown);
// The file's own colour and tone, carried onto the develop at the develop's own
// resolution. res is the camera's rendering less this develop's own low
// frequencies, one byte per channel with 0.5 as nothing-to-say, so adding it
// puts the camera's rendering back on the frame and leaves everything the
// develop has that the preview never did — the sensor's own detail — in the
// frame. pos is already the frame's own coordinate (the band is drawn at its
// own y offset, so the same pos is band-local only because crop.x is taken back
// off for the raw tap above), so one scale is the whole mapping.
float2 rp = pos * rscl;
o = clamp(o + (res.eval(rp).rgb - 0.5), 0.0, 1.0);
return half4(half3(o), 1.0);
}
`;
// The residual the develop above is carried by: the camera's own frame less the
// same frame as the develop left it, one byte per channel, 0.5 for a pixel the
// two already agree on. Both children are drawn at the size the residual is kept
// at, so neither needs a local matrix, and a minus b has to be able to go
// negative — hence the 0.5, which the develop takes back off.
const RESIDUAL_SKSL = `
uniform shader dev;
uniform shader ref;
half4 main(float2 p) {
return half4(half3(clamp(ref.eval(p).rgb - dev.eval(p).rgb + 0.5, 0.0, 1.0)), 1.0);
}
`;
@@ -467,19 +473,7 @@ function previewGrid(jpeg: Uint8Array, w: number, h: number, n = MATCH_GRID): Ui
// turn against the develop it was being fitted to, and the fit was rejected
// as folded (see previewMatch), which is why the Panasonic RW2 opened with no
// curve at all.
const targetAspect = w / h;
const bmpAspect = bmp.width() / bmp.height();
let cropW = bmp.width();
let cropH = bmp.height();
if (Math.abs(bmpAspect - targetAspect) > 0.01) {
if (bmpAspect > targetAspect) {
cropW = Math.round(bmp.height() * targetAspect);
} else {
cropH = Math.round(bmp.width() / targetAspect);
}
}
const cropX = Math.round((bmp.width() - cropW) / 2);
const cropY = Math.round((bmp.height() - cropH) / 2);
const { x: cropX, y: cropY, w: cropW, h: cropH } = previewCrop(bmp, w, h);
const surface = Skia.Surface.MakeOffscreen(n, n) ?? Skia.Surface.Make(n, n);
if (!surface) return null;
@@ -506,6 +500,144 @@ function previewGrid(jpeg: Uint8Array, w: number, h: number, n = MATCH_GRID): Ui
}
}
// The preview trimmed to the develop's own aspect, centred. What a body writes
// into the file is the frame the photographer saw, and it is the frame the
// sensor read; anything left over is a margin the develop does not have.
function previewCrop(bmp: any, w: number, h: number): { x: number; y: number; w: number; h: number } {
const aspect = w / h;
let cw = bmp.width();
let ch = bmp.height();
if (Math.abs(cw / ch - aspect) > 0.01) {
if (cw / ch > aspect) cw = Math.round(ch * aspect);
else ch = Math.round(cw / aspect);
}
return { x: Math.round((bmp.width() - cw) / 2), y: Math.round((bmp.height() - ch) / 2), w: cw, h: ch };
}
// The residual the develop is carried by, and how many frame pixels one of its
// own is worth.
export interface PreviewResidual {
img: any;
sx: number;
sy: number;
}
// The residual's own long side. It is a smooth map by construction — it holds no
// detail, that is the develop's half of the frame — so a preview wider than this
// is reduced rather than carried whole: the colour it carries survives the
// reduction, and the 24MP surface a NEF-sized preview would need does not.
const RESIDUAL_MAX = 2048;
// The one residual that says nothing: a pixel the camera's frame and this develop
// already agree on, for a file whose preview gave nothing to carry.
let NO_RESIDUAL: any = null;
function noResidual(): any {
if (!NO_RESIDUAL) {
// 0.5 in half, alpha 1 — the value the develop takes 0.5 back off.
const half = new Uint16Array([0x3800, 0x3800, 0x3800, 0x3c00]);
NO_RESIDUAL = Skia.Image.MakeImage(
{ width: 1, height: 1, colorType: Skia.ColorType.RGBA_F16, alphaType: Skia.AlphaType.Unpremul },
new Uint8Array(half.buffer),
8
);
}
return NO_RESIDUAL;
}
// What the camera's own rendering has to say about this develop: the whole
// difference between the two frames — the body's white balance, picture style,
// tone curve and colour, whatever the body is — carried at the preview's
// resolution as one byte a channel, 0.5 where the two already agree. The develop
// adds it back on the value it left, so the frame opens at the colour the
// photographer saw AND at the sensor's own resolution: the develop is only ever
// asked for the detail the preview never had.
//
// A mapping was tried here first — one cubic a channel, and a 3x3 under it before
// that — and neither is the camera's rendering: a picture style is a colour
// transform with a curve in it, and the best a per-channel cubic fitted between
// two grids could reach on the Fujifilm RAF this was reported on was dE00 6.04
// against the file's own preview. The residual reaches 0.54 on the same frame and
// 0.00 on every body that writes a full-size preview, because it is not a fit at
// all — it is the difference itself.
function previewResidual(first: any, jpeg: Uint8Array, w: number, h: number): PreviewResidual | null {
const bmp = Skia.Image.MakeImageFromEncoded(jpeg);
if (!bmp) return null;
let ref: any = null;
let dev: any = null;
let refShader: any = null;
let devShader: any = null;
let resShader: any = null;
let paint: any = null;
try {
const crop = previewCrop(bmp, w, h);
const cap = Math.min(1, RESIDUAL_MAX / Math.max(crop.w, crop.h));
const rw = Math.max(1, Math.round(crop.w * cap));
const rh = Math.max(1, Math.round(crop.h * cap));
// One small surface per frame, both at the residual's own size, so neither
// child needs a local matrix and the subtraction below is one tap for one tap.
const small = (draw: (canvas: any) => void): any => {
const surface = Skia.Surface.MakeOffscreen(rw, rh) ?? Skia.Surface.Make(rw, rh);
if (!surface) return null;
draw(surface.getCanvas());
surface.flush();
const shot = surface.makeImageSnapshot();
surface.dispose();
return shot;
};
ref = small((canvas) =>
canvas.drawImageRectCubic(
bmp,
Skia.XYWHRect(crop.x, crop.y, crop.w, crop.h),
Skia.XYWHRect(0, 0, rw, rh),
1 / 3,
1 / 3
)
);
dev = small((canvas) =>
canvas.drawImageRectCubic(
first,
Skia.XYWHRect(0, 0, w, h),
Skia.XYWHRect(0, 0, rw, rh),
1 / 3,
1 / 3
)
);
if (!ref || !dev) return null;
const effect = Skia.RuntimeEffect.Make(RESIDUAL_SKSL);
if (!effect) return null;
const tap = (img: any) =>
img.makeShaderOptions(
Skia.TileMode.Clamp,
Skia.TileMode.Clamp,
Skia.FilterMode.Nearest,
Skia.MipmapMode.None
);
devShader = tap(dev);
refShader = tap(ref);
resShader = effect.makeShaderWithChildren(new Float32Array(0), [devShader, refShader]);
paint = Skia.Paint();
paint.setShader(resShader);
const out = small((canvas) => canvas.drawRect(Skia.XYWHRect(0, 0, rw, rh), paint));
// The residual is read back at the develop's own resolution by a bilinear
// tap, so it is kept smooth: one byte a channel is finer than the shading
// it carries, which is the whole difference between two renderings.
return out ? { img: out, sx: rw / w, sy: rh / h } : null;
} catch {
return null;
} finally {
bmp.delete();
ref?.delete();
dev?.delete();
refShader?.delete();
devShader?.delete();
resShader?.delete();
paint?.delete();
}
}
// The camera -> sRGB matrix, applied to the WB'd sensor triple.
//
// `rgb_cam` is dcraw's own: it is the camera -> sRGB transform already, its rows
@@ -783,14 +915,23 @@ export async function developRaw(bytes: Uint8Array, fileName?: string): Promise<
const bandH = Math.max(1, Math.min(h, Math.floor(BAND_PIXELS / w)));
const f32 = new Float32Array(w * bandH * 4);
const half = new Uint16Array(w * bandH * 4);
const uniforms = new Float32Array(33);
// gain, mul, m0, m1, m2, crop (y offset of the band), then where the residual
// has to be read for the frame pixel this one draws.
const uniforms = new Float32Array(23);
uniforms[0] = SAMPLE_MAX / white;
uniforms.set([mul[0], mul[1], mul[2], 0, r0[0], r0[1], r0[2], 0, r1[0], r1[1], r1[2], 0, r2[0], r2[1], r2[2], 0], 1);
const develop = (tone: Float32Array | null) => {
const develop = (residual: PreviewResidual | null) => {
const surface = Skia.Surface.MakeOffscreen(w, h) ?? Skia.Surface.Make(w, h);
if (!surface) return null;
uniforms.set(tone ?? FLAT_TONE, 21);
uniforms[21] = residual ? residual.sx : 0;
uniforms[22] = residual ? residual.sy : 0;
const resChild = (residual?.img ?? noResidual()).makeShaderOptions(
Skia.TileMode.Clamp,
Skia.TileMode.Clamp,
Skia.FilterMode.Linear,
Skia.MipmapMode.None
);
for (let y0 = 0; y0 < h; y0 += bandH) {
const rows = Math.min(bandH, h - y0);
let o = 0;
@@ -814,7 +955,7 @@ export async function developRaw(bytes: Uint8Array, fileName?: string): Promise<
Skia.MipmapMode.None
);
uniforms[17] = y0;
const shader = effect.makeShaderWithChildren(uniforms, [child]);
const shader = effect.makeShaderWithChildren(uniforms, [child, resChild]);
const paint = Skia.Paint();
paint.setShader(shader);
surface.getCanvas().drawRect(Skia.XYWHRect(0, y0, w, rows), paint);
@@ -824,6 +965,7 @@ export async function developRaw(bytes: Uint8Array, fileName?: string): Promise<
child.delete();
band.delete();
}
resChild.delete();
const shot = surface.makeImageSnapshot();
surface.dispose();
@@ -839,22 +981,17 @@ export async function developRaw(bytes: Uint8Array, fileName?: string): Promise<
refJpeg = await cameraPreview(raw).catch(() => null);
}
if (refJpeg) {
try {
const devGrid = gridOf(first, MATCH_GRID);
const matchGrid = refGrid ?? previewGrid(refJpeg, w, h, MATCH_GRID);
if (devGrid && matchGrid) {
const fittedTone = toneMatch(devGrid, matchGrid, MATCH_GRID);
if (fittedTone) {
const second = develop(fittedTone);
if (second) {
finalShot = second;
first.dispose();
}
}
}
} catch {
// Keep first develop if tone matching fails
// The file's own rendering, carried onto this develop: what the camera wrote
// into the frame minus what the develop made of the same frame at that
// resolution, added back at the sensor's own. Nothing to fit and nothing to
// table — a body is matched against its own preview the moment it opens.
const residual = refJpeg ? previewResidual(first, refJpeg, w, h) : null;
if (residual) {
const second = develop(residual);
residual.img.delete();
if (second) {
finalShot = second;
first.dispose();
}
}