web: shoot through the live camera
The one path where the look is chosen before the picture exists: OPEN CAMERA grades the camera's own feed with the recipe in force, many times a second, and the shutter hands the studio the sensor's still under that same recipe. Preview and file differ in resolution only — the still is `takePhoto`'s own frame, not a copy of the small preview video, with `grabFrame` and a 2d copy of the element behind it for the browsers that ship no ImageCapture. The renderer gains two inputs for it: `sourceImage`, a picture the caller already decoded (re-encoding the camera's frame to JPEG only to decode it again would cost more than the whole render), and `drawTo`, which paints the finished picture instead of encoding it. One render is in flight at a time; a frame that arrives during one is dropped, so a slow device shows a lower frame rate rather than a queue of moments that have passed. The view flashed black on a phone. Setting width/height on a canvas resets its bitmap: measured on the preview, a resize leaves mean 0 until the next render lands, which on this box is 0.5s and on a phone more. The buffer was sized from every incoming frame, and a capture that renegotiates its resolution — which Chromium does when the page is too slow to consume its frames, and this pipeline runs ~2 fps at 720p under software GL — strobed black/picture at every switch. The buffer is now sized on the first frame and after that only when the frame's aspect changes: a same-aspect frame is scaled into it. Swapping a 1280x720 stream for a 640x360 one mid-view now leaves the buffer at 1280x720 with no black frame, and 640x360 renders at 6-13 fps instead of 2. The frames are read from a <video>, which is now IN the document (1px, behind the black backdrop) rather than detached: Safari draws blank frames from a detached video, which is the same black-between-pictures. It leaves the document with the view, and the tracks are stopped, so the camera light goes out. Probes: cam-smoke (feed painted, resolution, frame rate, a monochrome sim reaching the live frames, shutter into the studio, close, console clean), cam-renegotiate (no resize, no blank frame, status line on the frames), cam-close-flip (flip returns a picture; video gone on close).
This commit is contained in:
@@ -0,0 +1,250 @@
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { useI18n } from '../i18n/I18nProvider';
|
||||
import { CK, Skia } from '../engine/skiaShim';
|
||||
import { renderPhoto, type RenderAssets, type RenderOptions } from '../engine/exportEngine';
|
||||
import { cameraSupported, captureStill, closeCamera, openCamera, type CameraSession, type Facing } from '../engine/camera';
|
||||
import type { FrameId, GPSInfo, Recipe } from '../../shared/types';
|
||||
|
||||
// The live view: the camera's own feed, graded by the very pipeline that writes
|
||||
// the file, so whatever the user frames is what they get. Preview and file
|
||||
// differ in resolution only (see camera.ts) — never in look.
|
||||
//
|
||||
// One render is in flight at a time. A frame that arrives while the last is
|
||||
// still being graded is dropped rather than queued: the queue would grow by
|
||||
// itself on a slow device, and the honest answer there is a lower frame rate,
|
||||
// not twenty renders of a moment that has already passed.
|
||||
|
||||
// How often the frame-rate readout is refreshed. The number is a fact about the
|
||||
// device, not a control, so it only has to be current enough to read.
|
||||
const FPS_WINDOW_MS = 500;
|
||||
|
||||
export function CameraView({
|
||||
recipe,
|
||||
frameId,
|
||||
options,
|
||||
useGeotag,
|
||||
gpsInfo,
|
||||
assets,
|
||||
onShot,
|
||||
onClose,
|
||||
}: {
|
||||
recipe: Recipe;
|
||||
frameId: FrameId;
|
||||
options: RenderOptions;
|
||||
useGeotag: boolean;
|
||||
gpsInfo: GPSInfo | null;
|
||||
assets: RenderAssets;
|
||||
// One shot, full resolution, straight into the studio — where the same recipe
|
||||
// is already applied, on the still rather than on the video.
|
||||
onShot: (bytes: Uint8Array, name: string, blob: Blob) => Promise<void> | void;
|
||||
onClose: () => void;
|
||||
}) {
|
||||
const { t } = useI18n();
|
||||
const [facing, setFacing] = useState<Facing>('back');
|
||||
const [ready, setReady] = useState(false);
|
||||
const [fps, setFps] = useState(0);
|
||||
const [size, setSize] = useState<{ w: number; h: number } | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [shooting, setShooting] = useState(false);
|
||||
const canvasRef = useRef<HTMLCanvasElement>(null);
|
||||
const hostRef = useRef<HTMLDivElement>(null);
|
||||
const sessionRef = useRef<CameraSession | null>(null);
|
||||
const busyRef = useRef(false);
|
||||
// The buffer is sized once, on the first frame. See `blit`.
|
||||
const sizedRef = useRef(false);
|
||||
// One offscreen canvas, reused: where a frame that is not the buffer's size is
|
||||
// staged so it can be scaled into the fixed buffer.
|
||||
const scratchRef = useRef<HTMLCanvasElement | null>(null);
|
||||
// The loop runs off refs, so a recipe change (or a watermark added while the
|
||||
// camera is open) reaches the very next frame without restarting the camera.
|
||||
const liveRef = useRef({ recipe, frameId, options, useGeotag, gpsInfo });
|
||||
|
||||
liveRef.current = { recipe, frameId, options, useGeotag, gpsInfo };
|
||||
|
||||
// The composed picture, onto the visible canvas. A raw readback, not a JPEG:
|
||||
// this is the one place the bytes are wanted as pixels.
|
||||
//
|
||||
// The canvas buffer is sized on the first frame and, after that, only when the
|
||||
// frame's ASPECT changes. Setting width/height resets a canvas' bitmap, so a
|
||||
// resize is a black view until the next render lands — and a camera that
|
||||
// answers at another resolution mid-session (Chromium's capture renegotiates
|
||||
// when the page is too slow to consume its frames) would strobe between black
|
||||
// and picture at every switch. Same aspect, so the frame is scaled into the
|
||||
// buffer instead: the view goes soft rather than blank. A new aspect (the
|
||||
// other camera, usually) is worth the one clear — and the camera is starting
|
||||
// over there anyway.
|
||||
const blit = useCallback((image: any) => {
|
||||
const canvas = canvasRef.current;
|
||||
if (!canvas) return;
|
||||
const w = image.width();
|
||||
const h = image.height();
|
||||
const pixels = image.readPixels(0, 0, {
|
||||
width: w,
|
||||
height: h,
|
||||
colorType: CK.ColorType.RGBA_8888,
|
||||
alphaType: CK.AlphaType.Unpremul,
|
||||
colorSpace: CK.ColorSpace.SRGB,
|
||||
});
|
||||
if (!pixels) return;
|
||||
const clamped = new Uint8ClampedArray(pixels.buffer, pixels.byteOffset, pixels.length);
|
||||
const aspect = w / h;
|
||||
const resized =
|
||||
!sizedRef.current ||
|
||||
Math.abs(canvas.width / Math.max(1, canvas.height) - aspect) > 0.01;
|
||||
if (resized) {
|
||||
sizedRef.current = true;
|
||||
canvas.width = w;
|
||||
canvas.height = h;
|
||||
}
|
||||
// The status line reports the frames as they arrive, not the buffer: after a
|
||||
// renegotiation those are two different numbers, and the incoming one is the
|
||||
// resolution the preview is really running at.
|
||||
setSize((prev) => (prev && prev.w === w && prev.h === h ? prev : { w, h }));
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (!ctx) return;
|
||||
if (w === canvas.width && h === canvas.height) {
|
||||
ctx.putImageData(new ImageData(clamped, w, h), 0, 0);
|
||||
return;
|
||||
}
|
||||
const scratch = (scratchRef.current ??= document.createElement('canvas'));
|
||||
if (scratch.width !== w || scratch.height !== h) {
|
||||
scratch.width = w;
|
||||
scratch.height = h;
|
||||
}
|
||||
const sctx = scratch.getContext('2d');
|
||||
if (!sctx) return;
|
||||
sctx.putImageData(new ImageData(clamped, w, h), 0, 0);
|
||||
ctx.drawImage(scratch, 0, 0, w, h, 0, 0, canvas.width, canvas.height);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (!cameraSupported()) {
|
||||
setError(t('cam.unsupported'));
|
||||
return;
|
||||
}
|
||||
let raf = 0;
|
||||
let alive = true;
|
||||
let frames = 0;
|
||||
let since = performance.now();
|
||||
|
||||
openCamera(facing)
|
||||
.then((session) => {
|
||||
if (!alive) {
|
||||
closeCamera(session);
|
||||
return;
|
||||
}
|
||||
sessionRef.current = session;
|
||||
// The frames are read from a video element, and a DETACHED one is not
|
||||
// reliably painted: Safari draws blank frames from a video that is not in
|
||||
// the document, which is the same black-between-pictures a phone showed.
|
||||
// It lives behind the view, one pixel wide and never seen.
|
||||
hostRef.current?.appendChild(session.video);
|
||||
setReady(true);
|
||||
const tick = () => {
|
||||
raf = requestAnimationFrame(tick);
|
||||
const now = performance.now();
|
||||
frames++;
|
||||
if (now - since >= FPS_WINDOW_MS) {
|
||||
setFps(Math.round((frames * 1000) / (now - since)));
|
||||
frames = 0;
|
||||
since = now;
|
||||
}
|
||||
if (busyRef.current || !Skia) return;
|
||||
const video = session.video;
|
||||
if (video.readyState < 2) return;
|
||||
const source = Skia.Image.MakeImageFromCanvasImageSource(video);
|
||||
if (!source) return;
|
||||
busyRef.current = true;
|
||||
const live = liveRef.current;
|
||||
renderPhoto({
|
||||
sourceImage: source,
|
||||
recipe: live.recipe,
|
||||
frameId: live.frameId,
|
||||
useGeotag: live.useGeotag,
|
||||
gpsInfo: live.gpsInfo,
|
||||
assets,
|
||||
options: live.options,
|
||||
drawTo: blit,
|
||||
})
|
||||
.catch(() => undefined)
|
||||
.finally(() => {
|
||||
busyRef.current = false;
|
||||
});
|
||||
};
|
||||
raf = requestAnimationFrame(tick);
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
if (!alive) return;
|
||||
// A refused permission and an absent camera are different problems with
|
||||
// different answers, and the browser's own name is the honest one.
|
||||
setError(String((err as Error)?.name ?? err) === 'NotAllowedError' ? t('cam.denied') : t('cam.fail'));
|
||||
});
|
||||
|
||||
return () => {
|
||||
alive = false;
|
||||
cancelAnimationFrame(raf);
|
||||
closeCamera(sessionRef.current);
|
||||
sessionRef.current = null;
|
||||
};
|
||||
}, [facing, assets, blit, t]);
|
||||
|
||||
const shoot = useCallback(async () => {
|
||||
const session = sessionRef.current;
|
||||
if (!session || shooting) return;
|
||||
setShooting(true);
|
||||
setError(null);
|
||||
try {
|
||||
const shot = await captureStill(session);
|
||||
await onShot(shot.bytes, shot.name, shot.blob);
|
||||
} catch {
|
||||
setError(t('cam.shotFail'));
|
||||
setShooting(false);
|
||||
}
|
||||
}, [onShot, shooting, t]);
|
||||
|
||||
return (
|
||||
<div className="camera-backdrop" data-key="camera-view">
|
||||
{/* The stage and the file are one look; the live frame is shown whole, so
|
||||
the user composes against the geometry the export will use. */}
|
||||
<canvas ref={canvasRef} className="camera-feed" data-key="camera-feed" />
|
||||
{/* Where the <video> the frames are read from lives: in the document, out
|
||||
of sight (see the effect above). */}
|
||||
<div ref={hostRef} className="camera-source-host" aria-hidden />
|
||||
<div className="camera-bar">
|
||||
<span className="hint" data-key="camera-status">
|
||||
{/* The one number worth printing: the resolution the preview runs at,
|
||||
and how fast the device is actually grading it. */}
|
||||
{ready && size ? t('cam.status', { fps, w: size.w, h: size.h }) : t('cam.starting')}
|
||||
</span>
|
||||
<span className="spacer" />
|
||||
<button type="button" className="btn" data-key="camera-close" onClick={onClose}>
|
||||
{t('act.cancel')}
|
||||
</button>
|
||||
</div>
|
||||
{error ? <p className="err camera-err">{error}</p> : null}
|
||||
<div className="camera-deck">
|
||||
<button
|
||||
type="button"
|
||||
className="btn"
|
||||
data-key="camera-flip"
|
||||
title={t('cam.flip')}
|
||||
onClick={() => {
|
||||
setReady(false);
|
||||
setFacing((f) => (f === 'back' ? 'front' : 'back'));
|
||||
}}
|
||||
>
|
||||
⇄
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="shutter"
|
||||
data-key="camera-shutter"
|
||||
aria-label={t('cam.shot')}
|
||||
disabled={!ready || shooting}
|
||||
onClick={shoot}
|
||||
/>
|
||||
<span className="camera-spacer" />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -242,6 +242,7 @@ export function ImageStage({
|
||||
peek,
|
||||
onPeek,
|
||||
onFile,
|
||||
onCamera,
|
||||
cropEditing = false,
|
||||
cropRect,
|
||||
cropWH = null,
|
||||
@@ -289,6 +290,10 @@ export function ImageStage({
|
||||
peek: boolean;
|
||||
onPeek: (on: boolean) => void;
|
||||
onFile: (file: File) => void;
|
||||
// The other way a picture arrives: shot here, on the device's own camera, and
|
||||
// graded by the same pipeline the export uses (see CameraView). Absent on a
|
||||
// build with no camera to open.
|
||||
onCamera?: () => void;
|
||||
// Crop editor: shown over the un-cropped preview until APPLY commits it.
|
||||
cropEditing?: boolean;
|
||||
cropRect?: CropRect;
|
||||
@@ -1424,6 +1429,19 @@ export function ImageStage({
|
||||
>
|
||||
<strong>{t('drop.title')}</strong>
|
||||
<span>{t('drop.hint')}</span>
|
||||
{onCamera ? (
|
||||
<button
|
||||
type="button"
|
||||
className="btn primary"
|
||||
data-key="open-camera-empty"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
onCamera();
|
||||
}}
|
||||
>
|
||||
{t('cam.open')}
|
||||
</button>
|
||||
) : null}
|
||||
<p className="hint">{t('drop.demoLimit', { px: 2048 })}</p>
|
||||
</div>
|
||||
) : (
|
||||
@@ -1979,6 +1997,11 @@ export function ImageStage({
|
||||
<button type="button" className="btn" data-key="open-photo" onClick={() => inputRef.current?.click()}>
|
||||
{t('drop.replace')}
|
||||
</button>
|
||||
{onCamera ? (
|
||||
<button type="button" className="btn" data-key="open-camera" onClick={onCamera}>
|
||||
{t('cam.open')}
|
||||
</button>
|
||||
) : null}
|
||||
{onSaveOriginal ? (
|
||||
<button
|
||||
type="button"
|
||||
|
||||
Reference in New Issue
Block a user