web: shoot through the live camera

The one path where the look is chosen before the picture exists: OPEN CAMERA
grades the camera's own feed with the recipe in force, many times a second, and
the shutter hands the studio the sensor's still under that same recipe. Preview
and file differ in resolution only — the still is `takePhoto`'s own frame, not a
copy of the small preview video, with `grabFrame` and a 2d copy of the element
behind it for the browsers that ship no ImageCapture.

The renderer gains two inputs for it: `sourceImage`, a picture the caller
already decoded (re-encoding the camera's frame to JPEG only to decode it again
would cost more than the whole render), and `drawTo`, which paints the finished
picture instead of encoding it. One render is in flight at a time; a frame that
arrives during one is dropped, so a slow device shows a lower frame rate rather
than a queue of moments that have passed.

The view flashed black on a phone. Setting width/height on a canvas resets its
bitmap: measured on the preview, a resize leaves mean 0 until the next render
lands, which on this box is 0.5s and on a phone more. The buffer was sized from
every incoming frame, and a capture that renegotiates its resolution — which
Chromium does when the page is too slow to consume its frames, and this pipeline
runs ~2 fps at 720p under software GL — strobed black/picture at every switch.
The buffer is now sized on the first frame and after that only when the frame's
aspect changes: a same-aspect frame is scaled into it. Swapping a 1280x720
stream for a 640x360 one mid-view now leaves the buffer at 1280x720 with no
black frame, and 640x360 renders at 6-13 fps instead of 2.

The frames are read from a <video>, which is now IN the document (1px, behind
the black backdrop) rather than detached: Safari draws blank frames from a
detached video, which is the same black-between-pictures. It leaves the document
with the view, and the tracks are stopped, so the camera light goes out.

Probes: cam-smoke (feed painted, resolution, frame rate, a monochrome sim
reaching the live frames, shutter into the studio, close, console clean),
cam-renegotiate (no resize, no blank frame, status line on the frames),
cam-close-flip (flip returns a picture; video gone on close).

docker/frontend/src/engine/camera.ts

@@ -0,0 +1,120 @@
+// The live viewfinder, on the camera the browser hands over.
+//
+// Two resolutions, on purpose. The PREVIEW goes through the same render
+// pipeline as an export, many times a second, so it asks for 720p and no more:
+// the frame is only ever shown a few hundred pixels wide, and a phone holding a
+// graded shader loop at full rate is a phone that heats up until the browser
+// throttles it (camera_native_app_web.md, the thermal note). The STILL is the
+// opposite — `ImageCapture.takePhoto()` asks the sensor for the photo itself, so
+// what lands in the studio is not a frame of that same small video, which is the
+// one thing the web got to keep (md §4.2).
+//
+// What the browser will NOT give: the ultrawide or the tele lens, and the
+// phone's own HDR/Night processing. `facingMode` is the whole lens menu here
+// (md §3).
+
+export type Facing = 'back' | 'front';
+
+// `ideal`, never `exact`: a device with no 720p mode hands back its nearest,
+// where a refusal would cost the entire camera.
+const PREVIEW = { width: 1280, height: 720 };
+
+export interface CameraSession {
+  stream: MediaStream;
+  video: HTMLVideoElement;
+}
+
+// Camera is a secure-origin API: https, or localhost. A plain-http LAN address
+// (the NAS on `http://192.168.x.x:8090`) has `mediaDevices` undefined, and the
+// caller has to say so rather than fail silently.
+export function cameraSupported(): boolean {
+  return typeof navigator !== 'undefined' && !!navigator.mediaDevices?.getUserMedia;
+}
+
+export async function openCamera(facing: Facing): Promise<CameraSession> {
+  const stream = await navigator.mediaDevices.getUserMedia({
+    video: {
+      facingMode: { ideal: facing === 'front' ? 'user' : 'environment' },
+      width: { ideal: PREVIEW.width },
+      height: { ideal: PREVIEW.height },
+    },
+    audio: false,
+  });
+  const video = document.createElement('video');
+  video.muted = true;
+  video.playsInline = true;
+  video.setAttribute('playsinline', '');
+  video.srcObject = stream;
+  await video.play();
+  // play() resolves on the element's move to a playing state, which can still
+  // be a beat before there is a frame to draw.
+  if (video.readyState < 2) {
+    await new Promise<void>((resolve) => {
+      video.onloadeddata = () => resolve();
+    });
+  }
+  return { stream, video };
+}
+
+// The light must go out with the view: a track left running keeps the camera
+// indicator on and the battery draining. The element goes with it — it is in the
+// document while the view is open (see CameraView) and has no use outside it.
+export function closeCamera(session: CameraSession | null): void {
+  if (!session) return;
+  for (const track of session.stream.getTracks()) track.stop();
+  session.video.srcObject = null;
+  session.video.remove();
+}
+
+// The shutter. Three answers, best first: `takePhoto` is the sensor's own still
+// at full size, `grabFrame` a decoded frame of the same video, and a 2D copy of
+// the element whatever is on screen. Safari ships no ImageCapture at all, so a
+// viewfinder that could only shoot through it would show a picture and take
+// none.
+export async function captureStill(
+  session: CameraSession
+): Promise<{ bytes: Uint8Array; blob: Blob; name: string }> {
+  const track = session.stream.getVideoTracks()[0];
+  const pack = async (blob: Blob) => ({
+    bytes: new Uint8Array(await blob.arrayBuffer()),
+    blob,
+    name: stillName(),
+  });
+  for (const shoot of ['photo', 'frame'] as const) {
+    try {
+      const capture = new ImageCapture(track);
+      if (shoot === 'photo') return await pack(await capture.takePhoto());
+      // `grabFrame` is in the spec but not in TypeScript's DOM lib.
+      const frame = await (capture as unknown as { grabFrame(): Promise<ImageBitmap> }).grabFrame();
+      if (frame) return await pack(await drawJpeg(frame, frame.width, frame.height));
+    } catch {
+      /* not this browser, or not this lens: try the next answer down */
+    }
+  }
+  const el = session.video;
+  return await pack(await drawJpeg(el, el.videoWidth, el.videoHeight));
+}
+
+function drawJpeg(source: CanvasImageSource, w: number, h: number): Promise<Blob> {
+  const canvas = document.createElement('canvas');
+  canvas.width = Math.max(1, w);
+  canvas.height = Math.max(1, h);
+  const ctx = canvas.getContext('2d');
+  if (!ctx) return Promise.reject(new Error('no 2d context'));
+  ctx.drawImage(source, 0, 0, canvas.width, canvas.height);
+  return new Promise((resolve, reject) => {
+    canvas.toBlob(
+      (blob) => (blob ? resolve(blob) : reject(new Error('encode failed'))),
+      'image/jpeg',
+      0.95
+    );
+  });
+}
+
+// A shot is named the moment it is taken, because that is the only fact about it
+// the browser knows: it carries no EXIF and no original filename.
+function stillName(): string {
+  const d = new Date();
+  const p = (n: number) => String(n).padStart(2, '0');
+  return `camera-${d.getFullYear()}${p(d.getMonth() + 1)}${p(d.getDate())}-${p(d.getHours())}${p(d.getMinutes())}${p(d.getSeconds())}.jpg`;
+}

docker/frontend/src/engine/exportEngine.ts

@@ -96,13 +96,22 @@ export interface RenderOptions {
 }
 
 export interface RenderInput {
-  source: Uint8Array;
+  source?: Uint8Array;
   recipe: Recipe;
   frameId: FrameId;
   useGeotag?: boolean;
   gpsInfo?: GPSInfo | null;
   options?: RenderOptions;
   assets: RenderAssets;
+  // A picture the caller already decoded, instead of `source`'s bytes. The live
+  // camera has a new one every frame: re-encoding it to JPEG only for this to
+  // decode it again would cost more than the whole render.
+  sourceImage?: any;
+  // Paint the finished picture instead of encoding it, and hand it over here.
+  // The live view blits it and lets it go; the bytes of a JPEG would be thrown
+  // away thirty times a second. Called synchronously, with an image that is
+  // already dead once this returns.
+  drawTo?: (image: any) => void;
 }
 
 export interface RenderResult {
@@ -370,7 +379,7 @@ export async function renderPhoto(input: RenderInput): Promise<RenderResult | nu
     return obj as T;
   };
   try {
-    const skImage0 = Skia.Image.MakeImageFromEncoded(input.source);
+    const skImage0 = input.sourceImage ?? (input.source ? Skia.Image.MakeImageFromEncoded(input.source) : null);
     if (!skImage0) {
       console.error('Failed to parse source image');
       return null;
@@ -1056,6 +1065,13 @@ export async function renderPhoto(input: RenderInput): Promise<RenderResult | nu
       }
     }
 
+    // 10. The live view stops here: it wants the picture, not a file, and it
+    // says so with `drawTo` — no encode, no DPI patch, no bytes.
+    if (input.drawTo) {
+      input.drawTo(resultImage);
+      return { bytes: new Uint8Array(), width: resultImage.width(), height: resultImage.height() };
+    }
+
     let bytes = resultImage.encodeToBytes(Skia.ImageFormat.JPEG, 95);
     if (!bytes || bytes.length === 0) {
       console.error('Failed to encode image to JPEG');

docker/frontend/src/engine/skiaShim.ts

@@ -159,6 +159,11 @@ export async function initSkia(options: InitSkiaOptions = {}): Promise<any> {
         canvasKit.MakeImageFromEncoded(
           data instanceof Uint8Array ? data : new Uint8Array(data as ArrayBuffer)
         ),
+      // An image straight off a canvas source — what the camera's own <video>
+      // is. CanvasKit draws it to a canvas and reads it back; the live view
+      // needs no JPEG round trip to hand a frame to the renderer.
+      MakeImageFromCanvasImageSource: (src: CanvasImageSource) =>
+        canvasKit.MakeImageFromCanvasImageSource(src),
       // An image from bytes rather than from a file — the tone curve's 256x1
       // lookup table (see toneCurve.ts). RN Skia takes the factory here as
       // `Image.MakeImage(info, bytes, bytesPerRow)`; CanvasKit keeps that one