diff --git a/docker/frontend/src/App.tsx b/docker/frontend/src/App.tsx index 9c98b6c..da5382a 100644 --- a/docker/frontend/src/App.tsx +++ b/docker/frontend/src/App.tsx @@ -15,7 +15,7 @@ import { loadEngine } from './engine/assets'; import { renderPhoto, type RenderAssets } from './engine/exportEngine'; import { download, fileToBytes, readGps, readIso, resizedJpeg, scaledJpeg } from './engine/imageOps'; import { clearSession, forgetPhoto, loadPhoto, loadState, savePhoto, saveState } from './engine/session'; -import { upscaleJpeg } from './engine/superRes'; +import { preloadSuperRes, upscaleJpeg } from './engine/superRes'; import { api, type Photo, type SavedRecipe, type User } from './api'; import { TIERS } from './config/tiers'; import { DEFAULT_ADJUSTMENTS, DEFAULT_RECIPES, FILM_SIMS, filmSimToRecipe } from '../shared/utils/defaultRecipes'; @@ -1443,8 +1443,12 @@ export function Workspace() { promptPro(); return; } + // Some menu entry will be larger than this photo (the crop already counted), + // so an upscale is one tap away: fetch the runtime and the model while the + // visitor is still reading the sizes, instead of after they have chosen one. + if (editedLongest < (TIERS[pro ? 'user' : 'guest'].maxExportDim ?? 4096)) preloadSuperRes(); setChoosingExport(true); - }, [engine, source, pro, proLookInUse, promptPro]); + }, [engine, source, pro, proLookInUse, promptPro, editedLongest]); // The original file, handed back byte for byte — the untouched frame the // camera took, never the render. A page cannot write to the camera roll, so diff --git a/docker/frontend/src/engine/superRes.ts b/docker/frontend/src/engine/superRes.ts index 873c7b2..4df9e35 100644 --- a/docker/frontend/src/engine/superRes.ts +++ b/docker/frontend/src/engine/superRes.ts @@ -18,7 +18,8 @@ const WASM_DIR = '/wasm/ort/'; // than assembled at 4x and shrunk (which would cost the memory of both). const MODEL_SCALE = 4; // Source pixels per pass, and the margin the model sees but the draw discards — -// without it every tile boundary shows as a seam. +// without it every tile boundary shows as a seam. Counted in whatever the model +// is being fed, which is not always the photo's own pixels: see `feedScale`. const TILE = 256; const PAD = 8; @@ -53,6 +54,14 @@ function load(): Promise { const session = await ort.InferenceSession.create(MODEL_URL, { executionProviders: ['webgpu'] }).catch(() => ort.InferenceSession.create(MODEL_URL, { executionProviders: ['wasm'] }) ); + // One throwaway run through the graph: the runtime builds and caches its + // kernels on the first one, and that is a wait the visitor should spend + // looking at the export menu rather than at a progress bar. Sixteen + // pixels: inaudible next to the load, and it keeps the kernels out of the + // way of the tiles that follow. + await session + .run({ [session.inputNames[0]]: new ort.Tensor('float32', new Float32Array(3 * 16 * 16), [1, 3, 16, 16]) }) + .catch(() => {}); return { ort, session }; })().catch((err) => { // A failed load must not poison the next attempt. @@ -63,6 +72,14 @@ function load(): Promise { return loaded; } +// Start fetching the 32MB of runtime and model before the export that needs +// them, so the visitor waits for the pixels and not for the download. Called +// when the export menu opens — the only door onto an upscale — and ignored +// afterwards: every later call finds `load()` already in flight. +export function preloadSuperRes(): void { + void load().catch(() => {}); +} + // The photo's bytes, enlarged so its longest edge is `targetLongest`. The caller // only gets here when the target is larger than the picture (App decides), and // gets back the same bytes untouched when it is not. @@ -79,20 +96,49 @@ export async function upscaleJpeg( if (targetLongest <= longest) return bytes; const scale = targetLongest / longest; - const srcCanvas = new OffscreenCanvas(w, h); - const srcCtx = srcCanvas.getContext('2d', { willReadFrequently: true }); + // How much of the photo the model is handed, in photo pixels per pixel it + // reads. It answers with four pixels for every one it is given, so the only + // picture it ever has to read is `targetLongest / 4` across — hand it the + // whole photo instead and it invents four times the pixels being asked for, + // which the draw then throws three quarters of away on the way down to + // `targetLongest`. Same finished image, a quarter of the arithmetic: on a + // 2400px photo going to 4K that is 80 tiles of model for 20. + // + // The floor is the photo's own claim: below half its pixels the model is no + // longer enlarging the picture, it is drawing a new one from memory. + // The ceiling is the same idea from the other side — never hand it more + // pixels than the photo has, or the wait grows for nothing the eye can see. + const feedScale = Math.min(1, Math.max(scale / MODEL_SCALE, 0.5)); + const fw = Math.max(1, Math.round(w * feedScale)); + const fh = Math.max(1, Math.round(h * feedScale)); + const dstCanvas = new OffscreenCanvas(Math.max(1, Math.round(w * scale)), Math.max(1, Math.round(h * scale))); // Opaque: a partly covered edge pixel would otherwise survive as transparency // and the JPEG export flattens that onto black — a dark line down every seam. const dstCtx = dstCanvas.getContext('2d', { alpha: false }); - if (!srcCtx || !dstCtx) return bytes; - srcCtx.drawImage(bitmap, 0, 0); - const src = srcCtx.getImageData(0, 0, w, h); + if (!dstCtx) return bytes; + + const feedCanvas = new OffscreenCanvas(fw, fh); + const feedCtx = feedCanvas.getContext('2d', { willReadFrequently: true }); + if (!feedCtx) return bytes; + // 'high' matters here: this resample is the only one the photo gets before + // the model reads it, and a cheap one would hand it a soft picture to be + // sharp about. + feedCtx.imageSmoothingEnabled = true; + feedCtx.imageSmoothingQuality = 'high'; + feedCtx.drawImage(bitmap, 0, 0, fw, fh); + const src = feedCtx.getImageData(0, 0, fw, fh); const { ort, session } = await load(); const inputName = session.inputNames[0]; - const cols = Math.ceil(w / TILE); - const rows = Math.ceil(h / TILE); + const cols = Math.ceil(fw / TILE); + const rows = Math.ceil(fh / TILE); + // Destination pixels per fed pixel. Derived from the destination itself so + // the last row and column land exactly on its edge rather than a rounding + // short of it, and shared by neighbouring tiles so their boundary is the + // same number for both and nothing is left half-covered. + const stepX = dstCanvas.width / fw; + const stepY = dstCanvas.height / fh; let done = 0; onProgress?.({ done, total: cols * rows }); @@ -100,20 +146,20 @@ export async function upscaleJpeg( for (let tx = 0; tx < cols; tx++) { const x0 = tx * TILE; const y0 = ty * TILE; - const tw = Math.min(TILE, w - x0); - const th = Math.min(TILE, h - y0); + const tw = Math.min(TILE, fw - x0); + const th = Math.min(TILE, fh - y0); // The margin the model gets: full on the inside, clipped at the photo's // own edge, so the tensor covers whole pixels only. const left = Math.min(PAD, x0); const top = Math.min(PAD, y0); - const pw = tw + left + Math.min(PAD, w - (x0 + tw)); - const ph = th + top + Math.min(PAD, h - (y0 + th)); + const pw = tw + left + Math.min(PAD, fw - (x0 + tw)); + const ph = th + top + Math.min(PAD, fh - (y0 + th)); // NCHW, 0..1 RGB — what the model was trained to read. const input = new Float32Array(3 * pw * ph); const plane = pw * ph; for (let y = 0; y < ph; y++) { - const srow = ((y0 - top + y) * w + (x0 - left)) * 4; + const srow = ((y0 - top + y) * fw + (x0 - left)) * 4; for (let x = 0; x < pw; x++) { const s = srow + x * 4; input[y * pw + x] = src.data[s] / 255; @@ -158,10 +204,10 @@ export async function upscaleJpeg( tileCtx.putImageData(img, 0, 0); // Destination edges snapped to whole pixels: neighbouring tiles then share // the exact same boundary, so nothing is left half-covered (and half-black) - // where they meet. `scale` is rarely an integer, so `x0 * scale` alone + // where they meet. `stepX` is rarely an integer, so `x0 * stepX` alone // lands on a fraction and antialiases the seam. - const dx0 = Math.round(x0 * scale); - const dy0 = Math.round(y0 * scale); + const dx0 = Math.round(x0 * stepX); + const dy0 = Math.round(y0 * stepY); dstCtx.drawImage( tile, 0, @@ -170,8 +216,8 @@ export async function upscaleJpeg( ch, dx0, dy0, - Math.round((x0 + tw) * scale) - dx0, - Math.round((y0 + th) * scale) - dy0 + Math.round((x0 + tw) * stepX) - dx0, + Math.round((y0 + th) * stepY) - dy0 ); done++;