diff --git a/App.tsx b/App.tsx index b94484a..a88ddaf 100644 --- a/App.tsx +++ b/App.tsx @@ -29,6 +29,10 @@ import type { PickedPhotoAsset } from './src/utils/photoGps'; import { PARAM_DEFS } from './src/utils/paramDefs'; import { LITE_FAVORITE_LIMIT, LITE_MARK, LITE_RECIPE_LIMIT, proLookInUse } from './src/utils/entitlement'; import { initProUnlock, useProUnlock } from './src/utils/proUnlock'; +import { readJpegIso } from './src/utils/exifWrite'; +import { readPhotoBytes, devicePerf } from './src/utils/photoMeta'; +import { BURST_FRAMES, BURST_MIN_ISO, mergeBurst } from './src/utils/burstMerge'; +import { canBurst } from './src/utils/deviceFloor'; import UltraWide from './modules/recipescam-ultrawide'; import RecipescamExport from './modules/recipescam-export'; import VolumeShutter from './modules/recipescam-volume'; @@ -56,13 +60,6 @@ const CAMERA_SETTINGS_KEY = 'camera_settings'; // decides which engine ships. const NATIVE_EXPORT = process.env.EXPO_PUBLIC_NATIVE_EXPORT === '1'; -// B1: the shutter's CameraX capture mode. 'balanced' ships as the default — -// measured on the 12S Ultra at the same 3000x4000, a frame takes ~340 ms -// against ~540 ms for 'quality', with no resolution loss, because this device -// has ZSL (supportsSpeedQualityPrioritization) and 'balanced'/speed ride it. -// EXPO_PUBLIC_CAPTURE_MODE still overrides, but ONLY in dev bundles: Metro -// inlines EXPO_PUBLIC_*, the release HBC build does not (B0 finding) — never -// gate shipped behaviour on it. // How long a shutter may wait for a GPS fix before exporting without it. The // photo is never held hostage by the receiver: indoors a Balanced fix takes // 20s+ (measured 25s on the V2430), which is the whole reason a capture used to @@ -78,10 +75,6 @@ const loadRenderer = (native: boolean) => native ? import('./src/utils/nativeExport').then((m) => m.processAndExportPhotoNative) : import('./src/utils/exportEngine').then((m) => m.processAndExportPhoto); -const CAPTURE_MODE = - process.env.EXPO_PUBLIC_CAPTURE_MODE === 'speed' || process.env.EXPO_PUBLIC_CAPTURE_MODE === 'quality' - ? process.env.EXPO_PUBLIC_CAPTURE_MODE - : 'balanced'; export default function App() { const [mode, setMode] = useState<'camera' | 'library'>('camera'); @@ -342,7 +335,17 @@ export default function App() { // Full-resolution photo output — attached to the live in Viewfinder. // Photo capture is resolution-independent from the Skia preview output. const photoOutput = usePhotoOutput({ - qualityPrioritization: CAPTURE_MODE, + // LATENCY, measured: 'balanced' is CAPTURE_MODE_MINIMIZE_LATENCY (one + // frame, no CameraX still-image post-processing); 'quality' is + // CAPTURE_MODE_MAXIMIZE_QUALITY, which adds in-camera multi-frame noise + // reduction plus reprocessing. Same scene at ISO ~410 on the 12S Ultra, + // 3 shots per arm: in-camera START->onImageCaptured 291 ms vs 499 ms, + // tap->file on disk 797 ms vs 1025 ms (-22%), and the two JPEG sets were + // indistinguishable (flat-area sigma 1.08 vs 1.09, edge energy -3%). + // Dark scenes are handled by this app's own burst + merge below (ISO >= + // 800), so CameraX MFNR would only ever be a second, redundant pass. + // Revert to 'quality' if a high-ISO A/B ever shows merge losing to MFNR. + qualityPrioritization: 'balanced', }); // Second photo output for RAW DNG sidecars (RATIO panel → RAW on). Kept as // a live output only while rawEnabled — CameraX cannot bind two ImageCapture @@ -1495,6 +1498,56 @@ export default function App() { ) ).filePath; } + // Low light, hand-held: one frame is mostly sensor noise, and no OEM stack is + // reachable from here (the extension probe found the vendor pipelines closed to a + // third-party client). So take the frames and average them ourselves. Four ways + // out, checked before an extra exposure is spent: the LED is the light source + // (a burst would ride it while it recharges), the ultra-wide still came from the + // native Camera2 session, a held burst is the user asking for ONE frame, and RAW + // needs the second ImageCapture output CameraX will not bind alongside this one. + // canBurst: on a 1 GB floor device (deviceFloor.ts) three raws plus the + // alignment rasters are seconds of shutter lag, so the plain frame stands. + // Set when the low-light burst below starts its merge. The merge is not + // awaited on the shutter path: the export task awaits it instead, so the + // next shot is accepted while the frames are still being averaged. + let burstMerge: Promise | null = null; + if (!ultraWideShot && !opts?.burst && !rawEnabled && !(flashAvailable && flashMode !== 'off') && canBurst(devicePerf().ramMb)) { + try { + // ISO is the only signal this app can read for ambient light: the controller + // reports 0.0 for every still, so it comes off the file the shutter just + // wrote. No tag, no burst. + const jpeg = await readPhotoBytes(toUri(filePath)); + const iso = jpeg ? readJpegIso(jpeg) : null; + if (iso !== null && iso >= BURST_MIN_ISO) { + const frames = [filePath]; + for (let i = 1; i < BURST_FRAMES; i++) { + const extra = await photoOutput.capturePhotoToFile( + // No second shutter sound and no flash: three clicks for one photo + // reads as a malfunction. + { enableShutterSound: false, flashMode: 'off' }, + {}, + ); + frames.push(extra.filePath); + } + // Averaging three 12 MP frames costs ~1.4 s measured on the 12S Ultra + // (three decodes, two alignment stages, one draw pass, one JPEG + // encode) and all three frames are already in hand, so the merge + // rides the FIFO export queue below instead of the shutter path: + // the next shot is accepted as soon as the exposures land, and only + // the export of this photo waits on the average. A refused merge + // (null: drift too large, a frame unreadable) keeps the shutter frame. + // ponytail: a merge in the queue and a following capture can each + // hold a 12 MP frame (~48 MB); the FIFO serializes the merge against + // the render, not against the camera. Re-serialize if a low-RAM body + // starts swapping. + burstMerge = mergeBurst(frames); + } + } catch (burstError) { + // A failed burst costs two wasted exposures, never the photo. + console.warn('Low-light burst skipped:', burstError); + } + } + const sourceUri = toUri(filePath); // RAW DNG sidecar: fire the second exposure immediately after the JPEG @@ -1574,7 +1627,10 @@ export default function App() { frameWindowZoom: frameZoom, }; const paint = await loadRenderer(nativeExport); - return paint(sourceUri, recipeArgs, selectedFrame, useGeotag, captureGps, options); + // The burst average lands here, in the queue: the export waits for it, + // the shutter did not. sourceUri is the frame the shutter already wrote. + const source = burstMerge ? toUri((await burstMerge) ?? filePath) : sourceUri; + return paint(source, recipeArgs, selectedFrame, useGeotag, captureGps, options); }; enqueueExport(async () => { try { diff --git a/PLAN.md b/PLAN.md index f8b75fa..54c2611 100644 --- a/PLAN.md +++ b/PLAN.md @@ -82,6 +82,8 @@ Exit: user QA đạt → mặc định native; legacy chỉ fallback. - Nhắc: nếu npm install chạy lại → node_modules vision-camera về pristine, áp lại patch: `cd RecipesCam && git apply patches/vision-camera-shutter-silent.patch` (artifact = diff `HybridPhotoOutput.kt`: shutter OFF thật silent — bỏ OR mustPlayShutterSound). + `cd RecipesCam && git apply patches/vision-camera-isp-off.patch` + (artifact = 3 hunk, 2 file: `HybridPhotoOutput.kt` cho mọi still JPEG, `modules/recipescam-ultrawide/.../UltraWideModule.kt` cho đường 0.5x — cùng cặp cờ NOISE_REDUCTION_MODE + EDGE_MODE của ISP = OFF). ## Rủi ro / ghi chú - Local module phải bắt chước đúng cấu trúc expo package (đã đọc expo-media-library: `plugins { id 'com.android.library'; id 'expo-module-gradle-plugin' }`, expo-module.config.json khai báo class Kotlin; autolinking quét `./modules`). diff --git a/modules/recipescam-export/android/src/main/java/com/locphamtran/recipescamera/export/RecipescamExportModule.kt b/modules/recipescam-export/android/src/main/java/com/locphamtran/recipescamera/export/RecipescamExportModule.kt index 16c3df7..7b05e00 100644 --- a/modules/recipescam-export/android/src/main/java/com/locphamtran/recipescamera/export/RecipescamExportModule.kt +++ b/modules/recipescam-export/android/src/main/java/com/locphamtran/recipescamera/export/RecipescamExportModule.kt @@ -1,5 +1,6 @@ package com.locphamtran.recipescamera.export +import android.app.ActivityManager import android.content.ContentValues import android.content.Context import android.content.Intent @@ -206,6 +207,20 @@ class RecipescamExportModule : Module() { ) } + // Floor-device gate (src/utils/deviceFloor.ts): total RAM decides whether a + // full-resolution surface chain and the 3-frame burst fit. One cheap query + // and no permission; a platform that will not answer returns 0, which JS + // reads as "unknown" and keeps the full-quality path. + Function("devicePerf") { -> + val am = appContext.reactContext?.getSystemService(Context.ACTIVITY_SERVICE) as? ActivityManager + val mem = ActivityManager.MemoryInfo() + val known = if (am != null) { am.getMemoryInfo(mem); true } else false + mapOf( + "ramMb" to (if (known) (mem.totalMem / (1024L * 1024L)).toInt() else 0), + "cores" to Runtime.getRuntime().availableProcessors(), + ) + } + // P0 spike harness: copy a bundled drawable asset (e.g. "wallframe", the // 3117x4000 artwork — representative 12MP decode) into cacheDir and hand the // real file path back to JS, which then runs decodeEncodeAsync on it. This diff --git a/modules/recipescam-export/index.ts b/modules/recipescam-export/index.ts index 1c90161..1c5171f 100644 --- a/modules/recipescam-export/index.ts +++ b/modules/recipescam-export/index.ts @@ -105,6 +105,13 @@ export interface RecipescamExportModule { * codename, marketing name, manufacturer, model. Empty strings when absent. */ deviceInfo(): { device: string; marketName: string; manufacturer: string; model: string }; + /** + * Device facts the capture and render paths budget against (see + * src/utils/deviceFloor.ts): total RAM in MB and CPU core count. `ramMb` is 0 + * when the platform refused the read — callers treat 0 as "unknown", never as + * a floor device, so an unmeasurable phone keeps the full-quality path. + */ + devicePerf(): { ramMb: number; cores: number }; /** * The image a system share handed this launch (AndroidManifest diff --git a/src/utils/burstAlign.check.ts b/src/utils/burstAlign.check.ts new file mode 100644 index 0000000..9053d6d --- /dev/null +++ b/src/utils/burstAlign.check.ts @@ -0,0 +1,107 @@ +// Self-check for the burst alignment maths — the one part of the low-light +// merge that is decided by arithmetic rather than by Skia, and the part a wrong +// answer turns into a visibly doubled photo. Run it after touching +// burstAlign.ts (or the constants that feed it): +// node src/utils/burstAlign.check.ts +// Exits non-zero on the first failed expectation. + +import { bestShift, nearest2D, profilesOf, sadAt, type Pixels } from './burstAlign.ts'; + +function check(name: string, ok: boolean, detail = ''): void { + if (!ok) { + console.error(`FAIL ${name}${detail ? ` — ${detail}` : ''}`); + process.exit(1); + } + console.log(`ok ${name}`); +} + +// Deterministic structure (a hash, not Math.random) so a failure is repeatable. +function texture(w: number, h: number, seed: number): Pixels { + const px = new Uint8Array(w * h * 4); + for (let y = 0; y < h; y++) { + for (let x = 0; x < w; x++) { + const n = Math.sin(x * 12.9898 + y * 78.233 + seed) * 43758.5453; + const v = 96 + Math.floor((n - Math.floor(n)) * 128); + const p = (y * w + x) * 4; + px[p] = v; + px[p + 1] = v; + px[p + 2] = v; + px[p + 3] = 255; + } + } + return px; +} + +// `a` cut down to what a camera would see of `b` after a shift of (dx, dy). +function shifter(a: Pixels, w: number, h: number, dx: number, dy: number): Pixels { + const px = new Uint8Array(w * h * 4); + for (let y = 0; y < h; y++) { + for (let x = 0; x < w; x++) { + const sx = x - dx; + const sy = y - dy; + if (sx < 0 || sy < 0 || sx >= w || sy >= h) continue; + const d = (y * w + x) * 4; + const s = (sy * w + sx) * 4; + px[d] = a[s]; + px[d + 1] = a[s + 1]; + px[d + 2] = a[s + 2]; + px[d + 3] = 255; + } + } + return px; +} + +const W = 192; +const H = 128; +const ref = texture(W, H, 1); +const refProfile = profilesOf(ref, W, H); + +// 1) A known shift comes back out of the profiles, as the offset that draws the +// frame back onto the reference: content moved right by 7 is pulled left by 7. +{ + const cur = shifter(ref, W, H, 7, -4); + const curProfile = profilesOf(cur, W, H); + const dx = bestShift(refProfile.cols, curProfile.cols, 12); + const dy = bestShift(refProfile.rows, curProfile.rows, 12); + check('bestShift undoes a (7, -4) move', dx === -7 && dy === 4, `got (${dx}, ${dy})`); +} + +// 2) The 2D confirmation returns the translation, not the seed, when they agree. +{ + const cur = shifter(ref, W, H, -5, 3); + const curProfile = profilesOf(cur, W, H); + const seedX = bestShift(refProfile.cols, curProfile.cols, 12); + const seedY = bestShift(refProfile.rows, curProfile.rows, 12); + const best = nearest2D(ref, cur, W, H, seedX, seedY, 2); + check('nearest2D confirms it on the pixels', best.x === 5 && best.y === -3, `got (${best.x}, ${best.y})`); +} + +// 3) A featureless pair — an unreadable scene, or a flat wall in the dark — +// merges unshifted instead of being pushed somewhere by noise. +{ + const flat: Pixels = new Uint8Array(W * H * 4).fill(200); + for (let i = 3; i < flat.length; i += 4) flat[i] = 255; + const best = nearest2D(flat, flat, W, H, 9, 9, 2); + check('flat pair stays at (0, 0)', best.x === 0 && best.y === 0, `got (${best.x}, ${best.y})`); +} + +// 4) A candidate that keeps less than half the frame cannot win on a flat error. +{ + const a: Pixels = new Uint8Array(W * H * 4).fill(10); + const b: Pixels = new Uint8Array(W * H * 4).fill(10); + check('disjoint candidate rejected', sadAt(a, b, W, H, W, 0) === Infinity); + check('full-overlap candidate scored', sadAt(a, b, W, H, 0, 0) === 0); +} + +// 5) The fine stage searches around the coarse estimate: the true offset is +// found from a seed that is already close, and a seed that is wrong can only +// pull the answer as far as the window allows — never back to the truth. +{ + const cur = shifter(ref, W, H, 2, 0); + const curProfile = profilesOf(cur, W, H); + check('expect-centred search finds -2 from a seed of -2', bestShift(refProfile.cols, curProfile.cols, 1, -2) === -2); + const off = bestShift(refProfile.cols, curProfile.cols, 1, 5); + check('a wrong seed stays inside its window', Math.abs(off - 5) <= 1, `got ${off}`); +} + +console.log('\nburstAlign: all checks passed'); diff --git a/src/utils/burstAlign.ts b/src/utils/burstAlign.ts new file mode 100644 index 0000000..142df7e --- /dev/null +++ b/src/utils/burstAlign.ts @@ -0,0 +1,135 @@ +// Alignment maths for the low-light hand-held burst. Pure — no Skia, no React +// Native — so the check beside it runs on plain node: +// node src/utils/burstAlign.check.ts +// +// Frames taken within a second of each other differ by translation only (the +// zoom and the lens do not change between them), so the estimate is one shift +// per axis, read off the row and column luminance PROFILES of a downscaled +// frame. Correlating two profiles costs O(w+h) per candidate offset against +// O(w*h) for 2D block matching — the difference between a few hundred ms and +// minutes of JS thread for a 12MP burst. The 2D check then confirms the profile +// estimate against the actual pixels, so a periodic scene (blinds, railings, +// a tiled wall) cannot lock the estimator onto the wrong repetition — and (0,0) +// stays a candidate throughout, so a scene neither stage can read merges +// unshifted rather than wrongly shifted. + +export interface Profiles { + /** Mean luminance of each row, top to bottom. */ + rows: Float32Array; + /** Mean luminance of each column, left to right. */ + cols: Float32Array; +} + +/** RGBA pixels as they come out of Skia's readPixels (8-bit or float). */ +export type Pixels = Uint8Array | Float32Array; + +// Rec.709 luma on the unmanaged 8-bit values. A monotone luminance proxy is all +// a correlation needs — this is not a colour-managed Y. +const LUMA_R = 0.2126; +const LUMA_G = 0.7152; +const LUMA_B = 0.0722; + +export function profilesOf(rgba: Pixels, w: number, h: number): Profiles { + const rows = new Float32Array(h); + const cols = new Float32Array(w); + for (let y = 0; y < h; y++) { + const row = y * w * 4; + let sum = 0; + for (let x = 0; x < w; x++) { + const p = row + x * 4; + const l = rgba[p] * LUMA_R + rgba[p + 1] * LUMA_G + rgba[p + 2] * LUMA_B; + sum += l; + cols[x] += l; + } + rows[y] = sum / w; + } + for (let x = 0; x < w; x++) cols[x] /= h; + return { rows, cols }; +} + +/** + * The shift of `cur` against `ref`, in profile samples, searched over + * `expect ± max` and scored by mean absolute difference over the overlap. + * Mean, not sum, so a candidate that only matches on a handful of pixels is not + * favoured; overlaps shorter than half the signal are rejected outright. + * Convention: `ref[i] ≈ cur[i - shift]`. + */ +export function bestShift(ref: Float32Array, cur: Float32Array, max: number, expect = 0): number { + const center = Math.round(expect); + const lo = Math.max(-(cur.length - 1), center - max); + const hi = Math.min(ref.length - 1, center + max); + let best = center; + let bestErr = Infinity; + for (let s = lo; s <= hi; s++) { + const from = Math.max(0, s); + const to = Math.min(ref.length, cur.length + s); + if (to - from < ref.length / 2) continue; + let err = 0; + for (let i = from; i < to; i++) err += Math.abs(ref[i] - cur[i - s]); + err /= to - from; + if (err < bestErr) { + bestErr = err; + best = s; + } + } + return best; +} + +/** + * Mean absolute luminance difference between two downscaled frames at a + * candidate shift, over the overlap — the same convention as `bestShift` + * (`a[y][x] ≈ b[y - dy][x - dx]`). `Infinity` when the candidate keeps less than + * half the frame: a nearly disjoint pair would otherwise score a suspiciously + * low error on whatever flat area it did keep. + */ +export function sadAt(a: Pixels, b: Pixels, w: number, h: number, dx: number, dy: number): number { + const x0 = Math.max(0, dx); + const y0 = Math.max(0, dy); + const x1 = Math.min(w, w + dx); + const y1 = Math.min(h, h + dy); + if (x1 <= x0 || y1 <= y0) return Infinity; + if ((x1 - x0) * (y1 - y0) < (w * h) / 2) return Infinity; + let sum = 0; + for (let y = y0; y < y1; y++) { + const ra = (y * w + x0) * 4; + const rb = ((y - dy) * w + (x0 - dx)) * 4; + for (let x = x0; x < x1; x++) { + const pa = ra + (x - x0) * 4; + const pb = rb + (x - x0) * 4; + const la = a[pa] * LUMA_R + a[pa + 1] * LUMA_G + a[pa + 2] * LUMA_B; + const lb = b[pb] * LUMA_R + b[pb + 1] * LUMA_G + b[pb + 2] * LUMA_B; + sum += Math.abs(la - lb); + } + } + return sum / ((x1 - x0) * (y1 - y0)); +} + +/** + * Confirm a profile estimate on the pixels: search `±radius` around it, keeping + * (0,0) in the running, and return the best 2D match. This is what makes a + * profile lock onto a real translation rather than onto a repeated pattern — + * and what makes "no shift" win whenever the pixels do not support a shift. + */ +export function nearest2D( + ref: Pixels, + cur: Pixels, + w: number, + h: number, + seedX: number, + seedY: number, + radius: number +): { x: number; y: number } { + let best = { x: 0, y: 0 }; + let bestErr = sadAt(ref, cur, w, h, 0, 0); + for (let dy = seedY - radius; dy <= seedY + radius; dy++) { + for (let dx = seedX - radius; dx <= seedX + radius; dx++) { + if (dx === 0 && dy === 0) continue; + const err = sadAt(ref, cur, w, h, dx, dy); + if (err < bestErr) { + bestErr = err; + best = { x: dx, y: dy }; + } + } + } + return best; +} diff --git a/src/utils/burstMerge.ts b/src/utils/burstMerge.ts new file mode 100644 index 0000000..8251c4b --- /dev/null +++ b/src/utils/burstMerge.ts @@ -0,0 +1,227 @@ +// Low-light hand-held burst: N exposures of the same scene averaged into one +// JPEG, aligned first. This is the cheap end of what the stock camera does — an +// OEM stacks its own frames inside the HAL (and the extension probe showed no +// third-party client is served on the 12S/14 Ultra), so the only way to a +// multi-frame denoise from an app is to take the frames itself. The exposures +// come from the same photo output the shutter already uses, and the average +// buys about sqrt(N) off the sensor noise: ~1.7x for three frames, which is the +// difference between an ISO 1900 frame and one shot at ISO 700. +// +// Cost, all of it on the shutter path in the dark: N-1 extra exposures (~0.5s +// each in QUALITY mode), two decodes per frame (one for the alignment rasters, +// one for the merge draw — deliberately never more than one 12MP frame in +// memory at a time, see exportEngine's surface note) and one JPEG encode. + +import { Skia, AlphaType, BlendMode, ColorType, ImageFormat, type SkImage, type SkSurface } from '@shopify/react-native-skia'; +import { File } from 'expo-file-system'; +import * as FileSystem from 'expo-file-system/legacy'; +import { bestShift, nearest2D, profilesOf, type Profiles, type Pixels } from './burstAlign'; +import { stampExifOnFile } from './photoMeta'; + +// The ISO at which one frame stops being enough. Above it the shutter output is +// dominated by sensor noise — the 12S Ultra metered ISO 1918 on a dim indoor +// shot, where the stock camera stacks and the app could not. Below it there is +// nothing to win, so the plain (and much faster) capture stands. +// ponytail: one threshold, no shutter-speed term — CameraX exposes no exposure +// reading to a third-party app. Make it adaptive if ISO 800 ever misjudges a body. +export const BURST_MIN_ISO = 800; +// Total exposures. Three, not the four or five a HAL would stack: 1/3 is exactly +// representable as an 8-bit paint alpha (85/255), so the average carries no gain +// error, and two extra frames are as much hand-held drift as the merge tolerates. +export const BURST_FRAMES = 3; + +// Alignment scales: coarse (1/8) finds the offset over a wide range, fine (1/4) +// trims it to ±2 full-resolution pixels. Both are CPU surfaces — a downscaled +// raster is cheap there, and a GPU readback is not. +const COARSE_DIV = 8; +const FINE_DIV = 4; +const COARSE_RADIUS = 10; // ±80 px of a 3000 px frame +const COARSE_REFINE = 2; // 2D neighbourhood of the profile estimate, ±16 px +const FINE_RADIUS = 5; // ±20 px around the coarse estimate, in 1/4-scale pixels +// Motion beyond this share of the frame is a different picture, not drift: the +// intersection crop would eat the photo, and the average would ghost. +const MAX_SPREAD = 0.02; + +// GPU when a context exists (the merge draws three 12MP images), CPU otherwise — +// the same fallback exportEngine makes, for the same cold-start reason. +function surfaceFor(width: number, height: number): SkSurface | null { + return Skia.Surface.MakeOffscreen(width, height) ?? Skia.Surface.Make(width, height); +} + +// Capture outputs a bare path; expo-file-system's File wants a scheme. +const uriOf = (path: string): string => (/^[a-z][a-z0-9+.-]*:\/\//i.test(path) ? path : `file://${path}`); + +async function loadFrame(path: string): Promise { + try { + const bytes = await new File(uriOf(path)).bytes(); + const data = Skia.Data.fromBytes(bytes); + const image = Skia.Image.MakeImageFromEncoded(data); + data.dispose(); + return image; + } catch (e) { + console.warn('Burst frame decode failed:', e); + return null; + } +} + +// One downscaled RGBA raster of a frame, for the alignment stages. +function rasterize(src: SkImage, surface: SkSurface, w: number, h: number, paint: ReturnType): Pixels | null { + const canvas = surface.getCanvas(); + canvas.clear(Skia.Color('black')); + canvas.drawImageRect(src, Skia.XYWHRect(0, 0, src.width(), src.height()), Skia.XYWHRect(0, 0, w, h), paint); + return canvas.readPixels(0, 0, { width: w, height: h, colorType: ColorType.RGBA_8888, alphaType: AlphaType.Unpremul }); +} + +/** + * Average a hand-held burst of the same scene into one JPEG. + * + * `frames` are capture paths, the first one being the frame the shutter already + * took. Returns the cache path of the merged file, or null when the merge is not + * worth it or cannot be done — motion too large, a frame that will not decode, + * a frame of a different size — in which case the caller keeps the plain capture. + * The merged file carries the first frame's EXIF block, so the export still + * sees the camera and the capture instant it stamps into the finished photo. + */ +export async function mergeBurst(frames: string[]): Promise { + if (frames.length < 2) return null; + const owned: { dispose(): void }[] = []; + const keep = (item: T): T => { + owned.push(item); + return item; + }; + try { + // Frame 0 sets the geometry: CameraX returns every frame at the same size, + // so a frame that disagrees is not part of this burst. + const first = await loadFrame(frames[0]); + if (!first) return null; + const W = first.width(); + const H = first.height(); + first.dispose(); // geometry only: the loop decodes frame 0 for real + if (!(W > 0 && H > 0)) return null; + + const cw = Math.max(32, Math.round(W / COARSE_DIV)); + const ch = Math.max(32, Math.round(H / COARSE_DIV)); + const fw = Math.max(64, Math.round(W / FINE_DIV)); + const fh = Math.max(64, Math.round(H / FINE_DIV)); + const coarseSurface = Skia.Surface.Make(cw, ch); + const fineSurface = Skia.Surface.Make(fw, fh); + const accum = surfaceFor(W, H); + if (!coarseSurface || !fineSurface || !accum) { + coarseSurface?.dispose(); + fineSurface?.dispose(); + accum?.dispose(); + return null; + } + keep(coarseSurface); + keep(fineSurface); + keep(accum); + + const srcPaint = keep(Skia.Paint()); // an opaque source, unscaled + // The accumulator IS the output: each frame is drawn into it once its offset + // is known and released straight after, so at most one decoded 12MP frame + // (48MB) and one surface are alive at any moment. The 1/N alpha is what turns + // a plus-blend into an average — accumulating at full brightness would clip + // every highlight long before the third frame landed. + const addPaint = keep(Skia.Paint()); + addPaint.setBlendMode(BlendMode.Plus); + addPaint.setAlphaf(1 / frames.length); + accum.getCanvas().clear(Skia.Color('transparent')); + + let refCoarse: Profiles | null = null; + let refCoarsePixels: Pixels | null = null; + let refFine: Profiles | null = null; + const offsets: { x: number; y: number }[] = []; + + for (const path of frames) { + const image = await loadFrame(path); + if (!image) return null; + try { + if (image.width() !== W || image.height() !== H) return null; + const coarsePixels = rasterize(image, coarseSurface, cw, ch, srcPaint); + const finePixels = rasterize(image, fineSurface, fw, fh, srcPaint); + if (!coarsePixels || !finePixels) return null; + + let x = 0; + let y = 0; + if (refCoarse && refCoarsePixels && refFine) { + const coarseNow = profilesOf(coarsePixels, cw, ch); + const fineNow = profilesOf(finePixels, fw, fh); + // Profiles first: they are the only stage cheap enough to search a + // ±80 px range. Then the pixels confirm the result — that 2D pass is + // what keeps a periodic scene from locking onto the wrong repetition, + // and it keeps (0,0) as a candidate, so an unreadable pair merges + // unshifted rather than wrongly shifted. + const seed = nearest2D( + refCoarsePixels, + coarsePixels, + cw, + ch, + bestShift(refCoarse.cols, coarseNow.cols, COARSE_RADIUS), + bestShift(refCoarse.rows, coarseNow.rows, COARSE_RADIUS), + COARSE_REFINE, + ); + // 1/8-scale alignment is only good to ±8 full-resolution pixels; the + // 1/4-scale profiles, searched around it, trim that to ±2. + const seedScale = COARSE_DIV / FINE_DIV; + x = bestShift(refFine.cols, fineNow.cols, FINE_RADIUS, seed.x * seedScale) * FINE_DIV; + y = bestShift(refFine.rows, fineNow.rows, FINE_RADIUS, seed.y * seedScale) * FINE_DIV; + } else { + refCoarse = profilesOf(coarsePixels, cw, ch); + refCoarsePixels = coarsePixels; + refFine = profilesOf(finePixels, fw, fh); + } + + offsets.push({ x, y }); + accum.getCanvas().drawImage(image, x, y, addPaint); + } finally { + image.dispose(); + } + } + + const minX = Math.min(...offsets.map((o) => o.x)); + const maxX = Math.max(...offsets.map((o) => o.x)); + const minY = Math.min(...offsets.map((o) => o.y)); + const maxY = Math.max(...offsets.map((o) => o.y)); + if (maxX - minX > W * MAX_SPREAD || maxY - minY > H * MAX_SPREAD) return null; + + const accumulated = accum.makeImageSnapshot(); + if (!accumulated) return null; + keep(accumulated); + // The frames disagree at the borders by exactly the spread of their offsets, + // so keep the intersection: no edge is left carrying one frame's worth of + // exposure against the average of the rest. + const outW = W - (maxX - minX); + const outH = H - (maxY - minY); + const cropped = surfaceFor(outW, outH); + if (!cropped) return null; + keep(cropped); + cropped + .getCanvas() + .drawImageRect(accumulated, Skia.XYWHRect(minX, minY, outW, outH), Skia.XYWHRect(0, 0, outW, outH), srcPaint); + const merged = cropped.makeImageSnapshot(); + if (!merged) return null; + keep(merged); + + // Quality 90, the same the photo output produces: one more JPEG generation + // on a file that the export decodes and re-encodes anyway. + const bytes = merged.encodeToBytes(ImageFormat.JPEG, 90); + if (!bytes) return null; + const path = `${FileSystem.cacheDirectory}burst_${Date.now()}.jpg`; + new File(path).write(bytes); + await stampExifOnFile(path, {}, uriOf(frames[0])); + return path; + } catch (e) { + console.warn('Burst merge failed:', e); + return null; + } finally { + // Reverse creation order, like exportEngine: a snapshot is what holds the + // 48MB buffer, everything else is a handle onto it. + for (let i = owned.length - 1; i >= 0; i--) { + try { + owned[i].dispose(); + } catch { + // A buffer that is already gone must not cost the user the photo. + } + } + } +} diff --git a/src/utils/deviceFloor.check.ts b/src/utils/deviceFloor.check.ts new file mode 100644 index 0000000..846b8dc --- /dev/null +++ b/src/utils/deviceFloor.check.ts @@ -0,0 +1,42 @@ +// Runnable self-check: `node --no-warnings src/utils/deviceFloor.check.ts`. +// Covers the budget maths only — the RAM reading itself needs the device. +import assert from 'node:assert'; +import { + canBurst, + floorRaster, + FLOOR_EXPORT_PIXELS, + isFloorDevice, + maxExportPixels, +} from './deviceFloor.ts'; + +// Tier boundaries: unknown (0, the platform refused) is never a floor device, +// and the 4 GB mid-range keeps the full path. +assert.strictEqual(isFloorDevice(0), false, 'unknown RAM must keep full quality'); +assert.strictEqual(isFloorDevice(1024), true, 'Redmi Go / Nokia 1 / J2 Core'); +assert.strictEqual(isFloorDevice(2048), true, 'Go programme ceiling'); +assert.strictEqual(isFloorDevice(3072), false); +assert.strictEqual(isFloorDevice(12288), false, '12S Ultra'); + +// Caps follow the tier. +assert.strictEqual(maxExportPixels(0), Infinity); +assert.strictEqual(maxExportPixels(12288), Infinity); +assert.strictEqual(maxExportPixels(1024), FLOOR_EXPORT_PIXELS); +assert.strictEqual(canBurst(1024), false); +assert.strictEqual(canBurst(0), true); + +// The floor device's own 3264x2448 (8 MP) capture, and the 12S Ultra's 3000x4000. +const go = floorRaster(3264, 2448, FLOOR_EXPORT_PIXELS); +assert.ok(go.width < 3264, 'floor capture is resampled'); +assert.ok(go.width * go.height <= FLOOR_EXPORT_PIXELS, 'stays inside the cap'); +assert.ok(go.width * go.height > FLOOR_EXPORT_PIXELS * 0.9, 'and does not overshoot it'); +// Aspect is preserved to within a pixel on both orientations. +assert.ok(Math.abs(go.width / go.height - 3264 / 2448) < 0.002, '4:3 kept'); +const tall = floorRaster(3000, 4000, 4_000_000); +assert.ok(Math.abs(tall.width / tall.height - 3000 / 4000) < 0.002, '3:4 kept'); + +// Untouched when it already fits, and never zero-sized on a degenerate raster. +assert.deepStrictEqual(floorRaster(2000, 1500, FLOOR_EXPORT_PIXELS), { width: 2000, height: 1500 }); +assert.deepStrictEqual(floorRaster(1, 1, FLOOR_EXPORT_PIXELS), { width: 1, height: 1 }); +assert.deepStrictEqual(floorRaster(0, 0, 100), { width: 0, height: 0 }); + +console.log('deviceFloor.check: 17 assertions passed'); diff --git a/src/utils/deviceFloor.ts b/src/utils/deviceFloor.ts new file mode 100644 index 0000000..a181aaa --- /dev/null +++ b/src/utils/deviceFloor.ts @@ -0,0 +1,55 @@ +// The floor the whole pipeline is budgeted against: the cheapest Android that +// clears this app's minimum. What sets the minimum: minSdk 26 (below it +// vision-camera's NativeBufferHelper throws "HardwareBuffers require minSdk 26 +// or higher!" on every frame), a 32-bit armeabi-v7a build (the shipped APK +// carries both ABIs), Camera2 with EXIF, and Skia's GL backend, which links +// -lGLESv2 (see node_modules/@shopify/react-native-skia/android/CMakeLists.txt). +// Every API 26 phone clears the GL bar, so the floor is the Android Go, 1 GB +// class: Redmi Go (Snapdragon 425, Adreno 308, 5.8-7.2 GFLOPS / 5.3 GB/s, +// 8 MP), Nokia 1 (MT6737M, Mali-T720, 5 MP) and Galaxy J2 Core (Exynos 7570, +// 8 MP) — 4x Cortex-A53 at 1.1-1.4 GHz, 1 GB RAM, 8 GB storage, 480p-720p +// screen. Nothing weaker can install. +// +// What that floor affords, at 4 bytes per RGBA8888 pixel: +// 8 MP raster 32 MB -> the export's live surfaces alone = 96 MB +// one full-frame pass 64 MB of traffic @ 5.3 GB/s ~ 12 ms +// 4 slow A53 cores ~1/6 of a current flagship core +// Hence, here: one resample at the door instead of a full-resolution surface +// chain, and no 3-frame burst (six 8 MP decodes plus a CPU alignment on four +// slow cores is seconds of shutter lag on the device class least able to afford +// it). The live preview needs no cap — Viewfinder already forces the frame +// stream to the 4:3 size nearest 1280x960. +// +// No state and no native import: the RAM reading comes from +// photoMeta.devicePerf() and is passed in, so this module also runs under plain +// node (see deviceFloor.check.ts). +export const FLOOR_RAM_MB = 2048; +export const FLOOR_EXPORT_PIXELS = 4_000_000; + +// ramMb 0 = the platform would not say. Unknown is never a floor device: the app +// must not downgrade a phone it could not measure. +export const isFloorDevice = (ramMb: number): boolean => ramMb > 0 && ramMb <= FLOOR_RAM_MB; + +// Most pixels the export may paint. 4 MP (2336x1752) still prints 10x15 cm at +// 300 dpi and halves every surface behind it — the 12S Ultra's 12 MP capture is +// untouched, since it is not a floor device. +// ponytail: downscale rather than tile — tile the render the day a floor user +// needs the full 8 MP file. +export const maxExportPixels = (ramMb: number): number => + isFloorDevice(ramMb) ? FLOOR_EXPORT_PIXELS : Infinity; + +// ponytail: one capability, all or nothing — add a 2-frame CPU-lite merge if a +// floor device ever reports it can afford one. +export const canBurst = (ramMb: number): boolean => !isFloorDevice(ramMb); + +// Scale a raster down to at most `cap` pixels, keeping its aspect. Whole pixels +// only: the surface allocator and the EXIF stamp both read these back. +export function floorRaster( + w: number, + h: number, + cap: number +): { width: number; height: number } { + if (!(w > 0 && h > 0) || !(w * h > cap)) return { width: w, height: h }; + const s = Math.sqrt(cap / (w * h)); + return { width: Math.max(1, Math.round(w * s)), height: Math.max(1, Math.round(h * s)) }; +} diff --git a/src/utils/exifWrite.check.ts b/src/utils/exifWrite.check.ts new file mode 100644 index 0000000..06c69aa --- /dev/null +++ b/src/utils/exifWrite.check.ts @@ -0,0 +1,57 @@ +// Runnable self-check for the one reader the low-light burst gate leans on. +// expo-file-system has no host implementation, so this covers readJpegIso +// offline: node --no-warnings src/utils/exifWrite.check.ts +import { readJpegIso } from './exifWrite.ts'; + +// A 44-byte TIFF: IFD0 holds the Exif pointer, the Exif IFD holds 0x8827 — +// the two-hop shape a camera writes. `type` is the EXIF type id (3 SHORT, +// 4 LONG); a wrong-IFD or wrong-endian reader fails here. +function tiffWithIso(iso: number, type: 3 | 4, be = false): Uint8Array { + const t = new Uint8Array(44); + const u16 = (p: number, v: number) => (be ? (t[p] = v >> 8, t[p + 1] = v & 0xff) : (t[p] = v & 0xff, t[p + 1] = v >> 8)); + const u32 = (p: number, v: number) => + be + ? (t[p] = v >>> 24, t[p + 1] = (v >>> 16) & 0xff, t[p + 2] = (v >>> 8) & 0xff, t[p + 3] = v & 0xff) + : (t[p] = v & 0xff, t[p + 1] = (v >>> 8) & 0xff, t[p + 2] = (v >>> 16) & 0xff, t[p + 3] = v >>> 24); + t[0] = be ? 0x4d : 0x49; + t[1] = be ? 0x4d : 0x49; + u16(2, 42); + u32(4, 8); + u16(8, 1); // IFD0: one entry + u16(10, 0x8769); u16(12, 4); u32(14, 1); u32(18, 26); // -> Exif IFD at 26 + u32(22, 0); // no IFD1 + u16(26, 1); // Exif IFD: one entry + u16(28, 0x8827); u16(30, type); u32(32, 1); + if (type === 3) u16(36, iso); + else u32(36, iso); + u32(40, 0); + return t; +} + +function jpegWith(tiff: Uint8Array): Uint8Array { + const len = 2 + 6 + tiff.length; + const j = new Uint8Array(6 + len); // SOI + marker/len + the length's own payload + j[0] = 0xff; j[1] = 0xd8; // SOI + j[2] = 0xff; j[3] = 0xe1; // APP1 + j[4] = len >> 8; j[5] = len & 0xff; + j.set([0x45, 0x78, 0x69, 0x66, 0, 0], 6); + j.set(tiff, 12); + return j; +} + +const checks: [string, boolean][] = [ + ['SHORT ISO read back', readJpegIso(jpegWith(tiffWithIso(1234, 3))) === 1234], + ['LONG ISO read back', readJpegIso(jpegWith(tiffWithIso(2000, 4))) === 2000], + ['big-endian ISO read back', readJpegIso(jpegWith(tiffWithIso(800, 3, true))) === 800], + ['ISO 0 means unknown', readJpegIso(jpegWith(tiffWithIso(0, 3))) === null], + ['an unreadable file is not an ISO', readJpegIso(new Uint8Array([0xff, 0xd8, 0xff, 0xd9])) === null], + ['a non-JPEG is not an ISO', readJpegIso(new Uint8Array([1, 2, 3])) === null], +]; + +let bad = 0; +for (const [name, ok] of checks) { + console.log(`${ok ? 'ok ' : 'FAIL'} ${name}`); + if (!ok) bad++; +} +if (bad) throw new Error(`exifWrite: ${bad} check(s) failed`); +console.log('\nexifWrite: all checks passed'); diff --git a/src/utils/exifWrite.ts b/src/utils/exifWrite.ts index 14818d1..cb52e13 100644 --- a/src/utils/exifWrite.ts +++ b/src/utils/exifWrite.ts @@ -299,6 +299,30 @@ function findExifSegment(jpeg: Uint8Array): { start: number; end: number; tiff: return null; } +// --- read ------------------------------------------------------------------ + +/** + * The ISO the camera metered for this JPEG — EXIF 0x8827 + * (PhotographicSensitivity) — or null when the file carries none. The capture + * path cannot ask the camera: CameraX hands back a path and nothing else, and + * the controller's `iso` reads 0.0 for every still. So the low-light gate reads + * it back off the file it just wrote, and a file without the tag simply never + * bursts. + */ +export function readJpegIso(jpeg: Uint8Array): number | null { + const seg = findExifSegment(jpeg); + if (!seg) return null; + const iso = parseTiff(seg.tiff).exif.find((e) => e.tag === 0x8827); + if (!iso) return null; + // parseTiff normalizes every value to little-endian, so SHORT and LONG read + // with the same shifts. + const n = + iso.type === LONG || iso.type === SLONG + ? (iso.data[0] | (iso.data[1] << 8) | (iso.data[2] << 16) | (iso.data[3] << 24)) >>> 0 + : (iso.data[0] | (iso.data[1] << 8)) >>> 0; + return n > 0 && n < 1000000 ? n : null; // 0 and vendor noise both mean "unknown" +} + // --- serialize ------------------------------------------------------------- function serializeTiff(ifd0: Entry[], exif: Entry[], gps: Entry[], interop: Entry[]): Uint8Array { diff --git a/src/utils/exportEngine.ts b/src/utils/exportEngine.ts index 273e189..ed0d837 100644 --- a/src/utils/exportEngine.ts +++ b/src/utils/exportEngine.ts @@ -10,7 +10,8 @@ import { CINEMA_SKSL, getCinemaUniforms, cinemaIsActive } from './cinemaShader'; import { drawFrameOnCanvas, polaroidLayout, POLAROID_CARD, POLAROID_WIN_W, wallframeLayout, WALLFRAME_W, WALLFRAME_H, WALLFRAME_LAND_W, WALLFRAME_LAND_H } from './frameUtils'; import { patchJpegDpi } from './jpegDpi'; import { writeJpegExif } from './exifWrite'; -import { nextPhotoPath, readPhotoBytes, getDeviceIdentity } from './photoMeta'; +import { nextPhotoPath, readPhotoBytes, getDeviceIdentity, devicePerf } from './photoMeta'; +import { floorRaster, maxExportPixels } from './deviceFloor'; import { ensureMediaLibraryPermission } from './mediaPermission'; import RecipescamExport from '../../modules/recipescam-export'; @@ -246,6 +247,37 @@ export async function processAndExportPhoto( } let width = skImage.width(); let height = skImage.height(); + // 1c. Floor-device ceiling (see deviceFloor.ts). Everything below keeps more + // than one full-resolution surface alive — paint target, snapshot, sharpen — + // and one 8 MP raster is 32 MB, which is what a 1 GB Go phone pages or OOMs + // on mid-export. Resample once, here at the door, so every pass behind it + // (crop, tone, glow, grain, sharpen) runs on the smaller raster; the aspect + // is unchanged, so the ratio crop, frame layout and cover maths all stand. + // A surface that will not allocate leaves the raster alone: the full-size + // chain below is the one we are already on. + const capped = floorRaster(width, height, maxExportPixels(devicePerf().ramMb)); + if (capped.width !== width) { + const shrinkSurface = createSurface(capped.width, capped.height); + if (shrinkSurface) { + own(shrinkSurface); + shrinkSurface.getCanvas().drawImageRect( + skImage, + Skia.XYWHRect(0, 0, width, height), + Skia.XYWHRect(0, 0, capped.width, capped.height), + own(Skia.Paint()) + ); + const shrunk = shrinkSurface.makeImageSnapshot(); + if (shrunk) { + own(shrunk); + // Superseded, like the aspect crop above. + release(owned, shrinkSurface); + release(owned, skImage); + skImage = shrunk; + width = capped.width; + height = capped.height; + } + } + } // 1b. Aspect-ratio crop (composition): center-largest upright crop of the // requested ratio. Runs BEFORE any processing so frame/GPS/grain/sharpen // all apply to exactly the pixels the user framed. Wall Frame is exempt — diff --git a/src/utils/photoMeta.ts b/src/utils/photoMeta.ts index fa85e24..cb65a7e 100644 --- a/src/utils/photoMeta.ts +++ b/src/utils/photoMeta.ts @@ -3,6 +3,7 @@ import { File } from 'expo-file-system'; import * as FileSystem from 'expo-file-system/legacy'; import RecipescamExport from '../../modules/recipescam-export'; +import { isFloorDevice } from './deviceFloor'; import { writeJpegExif, SOFTWARE, type DeviceIdentity, type ExifStamp } from './exifWrite'; // RC_[_exported].jpg — the name the gallery shows for a saved @@ -50,6 +51,28 @@ export function getDeviceIdentity(): DeviceIdentity | null { return deviceIdentity; } +// Device facts the capture and render paths budget against (deviceFloor.ts): +// total RAM and CPU cores. Lives next to deviceInfo because that is where the +// native handle already is; read once, the numbers cannot change while the app +// runs. ramMb 0 = the platform refused, which deviceFloor reads as "not a floor +// device" so an unmeasurable phone keeps the full-quality path. +let devicePerfCache: { ramMb: number; cores: number } | null = null; +export function devicePerf(): { ramMb: number; cores: number } { + if (!devicePerfCache) { + try { + devicePerfCache = RecipescamExport.devicePerf(); + } catch (e) { + console.warn('devicePerf unavailable, full-quality path:', e); + devicePerfCache = { ramMb: 0, cores: 0 }; + } + // One line, so a floor device in the field can be confirmed from logcat: + // "adb logcat -s ReactNativeJS | grep floor". + const { ramMb, cores } = devicePerfCache; + console.info(`devicePerf: ramMb=${ramMb} cores=${cores} floor=${isFloorDevice(ramMb)}`); + } + return devicePerfCache; +} + // Stamp EXIF into a file already on disk (the native engine writes the JPEG // itself, so those bytes never pass through JS). Never throws — a metadata // failure must not cost the user the photo. diff --git a/tsconfig.json b/tsconfig.json index b9567f6..ca7790f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,6 +1,9 @@ { "extends": "expo/tsconfig.base", "compilerOptions": { - "strict": true + "strict": true, + // burstAlign.check.ts runs under node's own type stripping, where the import + // must name the .ts file. Type-only flag; expo base already sets noEmit. + "allowImportingTsExtensions": true } }