diff --git a/docker/frontend/shared/utils/toneShader.ts b/docker/frontend/shared/utils/toneShader.ts index 7390e6b..4ea26ec 100644 --- a/docker/frontend/shared/utils/toneShader.ts +++ b/docker/frontend/shared/utils/toneShader.ts @@ -544,65 +544,62 @@ export const CLARITY_GAIN = 3.2; // The patch AVERAGE this pass used to read instead (the bilateral reference) // called every patch hazy, so the positive end ground the frame down instead of // taking haze out. `air` is the atmospheric light the caller estimated from the -// frame, `step` one tap of the patch in the caller's own pixels — a fraction of -// the frame's width, so the preview and the file look at the same neighbourhood -// (DEHAZE_PATCH_STEP). +// frame — brightest dark-channel pixel of a copy of it, the doc's 0.1% answer in +// one readback (exportEngine's atmosphericLight). // -// `amount` is signed. Positive pushes the transmission below 1 and -// `J = (I - A)/t + A` takes the scattered light out; negative pushes it above 1 -// and the same expression scatters light back in, which is what a negative -// DEHAZE is for. The floor keeps a flat sky from dividing by zero, and the -// ceiling is the largest amount the knob can ask for either way. -// Ray marching the doc's A estimate would need the histogram; the caller reads a -// 32x32 copy of the frame instead and takes its brightest dark-channel pixel — -// the same 0.1% answer, in one readback (see exportEngine's atmosphericLight). +// The pass itself is now only the doc's last line, `J = (I - A)/t + A`, on a +// transmission the caller has already solved for. `dark` is what the caller +// hands in as an image: the dark channel itself, read off a small copy of the +// frame and interpolated back up, which is the smoothing the prior wants — see +// the caller's dehazeDarkChannel for why it cannot be had from a patch read out +// per pixel. Its cell is the patch, so a value per cell is a value per patch. +// +// What travels as an image is the dark channel and not t on purpose. A channel is +// eight bits, so it can only carry 0..1 — and t is 1 + 0.95 at the negative end +// of the knob, which would arrive here clipped to 1 and turn "put the scattered +// light back" into a pass that does nothing. The dark channel is 0..1 by +// construction, and the signed amount stays a uniform where it costs no range. +// +// Signedness is then in the expression. Positive folds t below 1 and takes the +// scattered light out; negative folds it above 1 and the same expression scatters +// it back in, which is what a negative DEHAZE is for. The floor keeps a flat sky +// from dividing by zero, and the ceiling is the largest amount the knob can ask +// for either way. export const DEHAZE_FLOOR_T = 0.1; export const DEHAZE_MAX_OMEGA = 0.95; -// The dark channel's patch, and the two ends of the transmission t. The patch is -// `taps` samples out at `step` each — two taps at 0.625% of the frame's width is -// a 2.5%-wide neighbourhood, the DCP's own 15-pixel patch on a 600-pixel frame -// and the same fraction of a 4000-pixel export. Five by five samples rather than +// The patch the MASK's DEHAZE reads out of its own frame (gradientMask.ts) — +// `taps` samples out at `step` each, two taps at 0.625% of the frame's width, a +// 2.5%-wide neighbourhood: the DCP's own 15-pixel patch on a 600-pixel frame and +// the same fraction of a 4000-pixel export. Five by five samples rather than // fifteen by fifteen because the doc's 225 reads per pixel is what // CLARITY_BLUR_SKSL above already refused, and the prior only needs a patch the -// haze is flat over. +// haze is flat over. The frame-wide pass reads no patch at all any more. export const DEHAZE_PATCH_TAPS = 2; export const DEHAZE_PATCH_STEP = 0.00625; export const DEHAZE_SKSL = ` uniform shader img; +uniform shader dark; uniform float3 air; -uniform float amount; uniform float floorT; -uniform float stepPx; +uniform float maxT; +uniform float amount; vec4 main(vec2 xy) { vec3 c = clamp(img.eval(xy).rgb, 0.0, 1.0); vec3 a = max(air, vec3(0.05)); - float dark = 1.0; - for (int j = -${DEHAZE_PATCH_TAPS}; j <= ${DEHAZE_PATCH_TAPS}; j++) { - for (int i = -${DEHAZE_PATCH_TAPS}; i <= ${DEHAZE_PATCH_TAPS}; i++) { - vec3 p = clamp(img.eval(xy + vec2(float(i), float(j)) * stepPx).rgb, 0.0, 1.0); - dark = min(dark, min(min(p.r / a.r, p.g / a.g), p.b / a.b)); - } - } - float t = clamp(1.0 - amount * clamp(dark, 0.0, 1.0), floorT, 1.0 + ${DEHAZE_MAX_OMEGA}); + float d = clamp(dark.eval(xy).r, 0.0, 1.0); + float t = clamp(1.0 - amount * ${DEHAZE_MAX_OMEGA} * d, floorT, maxT); return vec4(clamp((c - a) / t + a, 0.0, 1.0), 1.0); } `; export function dehazeUniformArray( air: [number, number, number], - amount: number, - stepPx: number + amount: number ): number[] { 'worklet'; - return [air[0], air[1], air[2], amount * DEHAZE_MAX_OMEGA, DEHAZE_FLOOR_T, stepPx]; -} - -// One tap of that patch in the pixels of a frame this wide. -export function dehazePatchStep(width: number): number { - 'worklet'; - return Math.max(1, width * DEHAZE_PATCH_STEP); + return [air[0], air[1], air[2], DEHAZE_FLOOR_T, 1 + DEHAZE_MAX_OMEGA, amount]; } export interface ToneUniforms { diff --git a/docker/frontend/src/engine/exportEngine.ts b/docker/frontend/src/engine/exportEngine.ts index 4737412..21a8d7c 100644 --- a/docker/frontend/src/engine/exportEngine.ts +++ b/docker/frontend/src/engine/exportEngine.ts @@ -33,7 +33,6 @@ import { CLARITY_GAIN, DEHAZE_SKSL, dehazeUniformArray, - dehazePatchStep, getToneUniforms, toneIsActive, toneUniformArray, @@ -381,6 +380,11 @@ function spatialReference(surface: any, w: number, h: number): any | null { // answer is the pixel whose DARKEST channel is brightest, so the 4096-pixel copy // only has to be searched for that one. Null when the readback is unavailable — // the caller then leaves the pass out rather than guessing a value. +// Where a pixel stops being evidence of A, and the share of the survivors that +// speaks for it (see the note inside). +const DEHAZE_AIR_BLOWN = 0.9; +const DEHAZE_AIR_FRACTION = 0.02; + function atmosphericLight(surface: any, w: number, h: number): [number, number, number] | null { const n = 32; const small = createSurface(n, n); @@ -407,19 +411,39 @@ function atmosphericLight(surface: any, w: number, h: number): [number, number, colorSpace: Skia.ColorSpace.SRGB, }); if (!px) return null; - let best = -1; - let air: [number, number, number] = [1, 1, 1]; + // A blown pixel's dark channel is 1 whatever the haze is — white cloth the + // camera clipped reads exactly like white haze — so a frame with a white + // anything in it answered A = [1, 1, 1]. That is the worst A to hand + // `J = (I - A)/t + A`: every pixel below 1 goes negative on some channel and + // is clipped back channel by channel, which is the lilac cast on white cloth + // and the bright rim around every dark detail (one pixel over, the patch + // holds a shadow and t is 1 again). + // + // So the pixels that can be evidence of the light are the ones short of + // white, and A is the mean of the brightest of those — the doc's 0.1% is one + // pixel of this 32x32 copy, which a single specular dot would own. + const evidence: Array<[number, number, number]> = []; for (let i = 0; i < n * n; i++) { - const r = px[i * 4] / 255; - const g = px[i * 4 + 1] / 255; - const b = px[i * 4 + 2] / 255; - const dark = Math.min(r, g, b); - if (dark > best) { - best = dark; - air = [r, g, b]; - } + const rgb: [number, number, number] = [ + px[i * 4] / 255, + px[i * 4 + 1] / 255, + px[i * 4 + 2] / 255, + ]; + if (Math.min(rgb[0], rgb[1], rgb[2]) < DEHAZE_AIR_BLOWN) evidence.push(rgb); } - return air; + // A frame with nothing short of white in it is all haze: the doc's answer + // would be A = 1 and the inversion would be all clipping, so DEHAZE stays off + // instead (the same null the readback failure returns). + if (evidence.length < 2) return null; + evidence.sort((l, r) => Math.min(r[0], r[1], r[2]) - Math.min(l[0], l[1], l[2])); + const keep = Math.max(1, Math.round(evidence.length * DEHAZE_AIR_FRACTION)); + const air: [number, number, number] = [0, 0, 0]; + for (let i = 0; i < keep; i++) { + air[0] += evidence[i][0]; + air[1] += evidence[i][1]; + air[2] += evidence[i][2]; + } + return [air[0] / keep, air[1] / keep, air[2] / keep]; } catch { // No readback (a surface that refuses one, a shader that failed): DEHAZE // stays off for this render. @@ -430,6 +454,132 @@ function atmosphericLight(surface: any, w: number, h: number): [number, number, } } +// DEHAZE's dark channel, the prior's own quantity, built as an image for +// DEHAZE_SKSL to sample — the pass itself turns it into t. +// +// The pass used to take the dark channel's minimum inline, over five taps spaced +// 0.625% of the frame's width apart — one tap every 7 pixels of a 1067-pixel +// preview. A dark detail thinner than that spacing sat between two taps on one +// row of the frame and under a tap on the next, so t swung between "this patch +// holds a shadow, leave it alone" and "this patch is all haze, divide hard" with +// a 7-pixel period around every dark thing in the photo. A rising DEHAZE drew +// that period as bright copies of the detail, stacked beside it. +// +// Reading the dark channel off a small copy of the frame is what removes it: one +// cell of that copy already covers a whole patch, so the neighbourhood is +// covered rather than sampled, and the bilinear upscale interpolates it back with +// no period left in it. The box passes after that are the doc's soft matting in +// the one form that is free here — the map is smoothed, not the pixels. +// +// The dark channel and not t is what goes into that image, because a channel is +// eight bits and only carries 0..1: t is 1 + 0.95 at the negative end of the +// knob, and would arrive clipped to 1, turning "scatter the light back in" into a +// pass that does nothing at all. The dark channel is 0..1 by construction, and +// the signed amount stays a uniform, where it costs no range. +const DEHAZE_MAP_SIZE = 64; +const DEHAZE_MAP_TAPS = 1; // patch reach, in cells of that copy +const DEHAZE_MAP_BLUR = 1; // box radius, in cells of that copy + +// One separable box pass over the copy's own grid. +function boxBlurMap(src: Float32Array, n: number, r: number): Float32Array { + const mid = new Float32Array(n * n); + const out = new Float32Array(n * n); + const edge = (v: number) => Math.min(n - 1, Math.max(0, v)); + for (let y = 0; y < n; y++) { + for (let x = 0; x < n; x++) { + let sum = 0; + for (let i = -r; i <= r; i++) sum += src[y * n + edge(x + i)]; + mid[y * n + x] = sum / (2 * r + 1); + } + } + for (let y = 0; y < n; y++) { + for (let x = 0; x < n; x++) { + let sum = 0; + for (let j = -r; j <= r; j++) sum += mid[edge(y + j) * n + x]; + out[y * n + x] = sum / (2 * r + 1); + } + } + return out; +} + +// Null when the copy cannot be read: the caller then leaves DEHAZE out for this +// render rather than correcting with a t it made up (atmosphericLight's own +// answer to the same failure). +function dehazeDarkChannel( + surface: any, + w: number, + h: number, + air: [number, number, number] +): any | null { + const n = DEHAZE_MAP_SIZE; + const small = createSurface(n, n); + if (!small) return null; + let snap: any = null; + let paint: any = null; + try { + flush(surface); + snap = surface.makeImageSnapshot(); + if (!snap) return null; + paint = Skia.Paint(); + small.getCanvas().drawImageRect( + snap, + Skia.XYWHRect(0, 0, w, h), + Skia.XYWHRect(0, 0, n, n), + paint + ); + flush(small); + const px = small.getCanvas().readPixels(0, 0, { + width: n, + height: n, + colorType: Skia.ColorType.RGBA_8888, + alphaType: Skia.AlphaType.Unpremul, + colorSpace: Skia.ColorSpace.SRGB, + }); + if (!px) return null; + const a = [Math.max(air[0], 0.05), Math.max(air[1], 0.05), Math.max(air[2], 0.05)]; + const dark = new Float32Array(n * n); + for (let y = 0; y < n; y++) { + for (let x = 0; x < n; x++) { + let d = 1; + for (let j = -DEHAZE_MAP_TAPS; j <= DEHAZE_MAP_TAPS; j++) { + const cy = Math.min(n - 1, Math.max(0, y + j)); + for (let i = -DEHAZE_MAP_TAPS; i <= DEHAZE_MAP_TAPS; i++) { + const o = (cy * n + Math.min(n - 1, Math.max(0, x + i))) * 4; + d = Math.min(d, px[o] / 255 / a[0], px[o + 1] / 255 / a[1], px[o + 2] / 255 / a[2]); + } + } + dark[y * n + x] = Math.min(1, Math.max(0, d)); + } + } + const smooth = boxBlurMap(boxBlurMap(dark, n, DEHAZE_MAP_BLUR), n, DEHAZE_MAP_BLUR); + // The dark channel rides the red channel and every byte is opaque, so the + // image is its own premultiplied answer and needs no range the shader decodes. + const bytes = new Uint8Array(n * n * 4); + for (let i = 0; i < n * n; i++) { + const v = Math.min(255, Math.max(0, Math.round(smooth[i] * 255))); + bytes[i * 4] = v; + bytes[i * 4 + 1] = v; + bytes[i * 4 + 2] = v; + bytes[i * 4 + 3] = 255; + } + return Skia.Image.MakeImage( + { + width: n, + height: n, + colorType: Skia.ColorType.RGBA_8888, + alphaType: Skia.AlphaType.Premul, + }, + bytes, + n * 4 + ); + } catch { + return null; + } finally { + disposeAll([paint, snap].filter(Boolean) as SkDisposable[]); + small.dispose(); + } +} + // An image as a child shader — what every effect here wants instead of a paint. // The caller owns the result (the renderer's `own`). function imageShaderChild(image: any): any { @@ -778,22 +928,64 @@ export async function renderPhoto(input: RenderInput): Promise { - const a = airOf(); - if (!a) return null; - const shader = dehazeEffect.makeShaderWithChildren( - dehazeUniformArray(a, dehazeAmount, dehazePatchStep(width)), - [own(imageShaderChild(snap))] - ); - return shader ? own(shader) : null; - }); + if (dehazeAmount !== 0 && effect) { + // The dark channel is read off the frame the pass is about to overwrite, + // like the light and for the same reason: after the pass the pixels are the + // corrected ones and the dark channel they would give is not the haze's. + const a = airOf(); + const darkMap = a ? own(dehazeDarkChannel(surface, width, height, a)) : null; + if (a && darkMap) { + replaceThrough(canvas, surface, width, height, (snap) => { + const shader = dehazeEffect.makeShaderWithChildren( + dehazeUniformArray(a, dehazeAmount), + [ + own(imageShaderChild(snap)), + own( + darkMap.makeShaderOptions( + Skia.TileMode.Clamp, + Skia.TileMode.Clamp, + Skia.FilterMode.Linear, + Skia.MipmapMode.None, + // A map this small only covers the frame if it is told to. With + // no matrix the image's own texels ARE the shader's coordinates, + // so `dark.eval` reads the map 1:1: the top-left 64 pixels of + // the frame get the real dark channel and every pixel past them + // clamps onto the map's last texel — one constant t over the + // whole photo, i.e. a global inversion instead of a dehaze. + // CanvasKit reads this matrix as the map's own pixels to the + // frame's (measured: `scaled(n/w)` clamps the same way, the + // reciprocal lands on the ideal ramp), hence frame over map, + // as a plain 3x3 in SkMatrix's own order. + [width / DEHAZE_MAP_SIZE, 0, 0, 0, height / DEHAZE_MAP_SIZE, 0, 0, 0, 1] + ) + ), + ] + ); + return shader ? own(shader) : null; + }); + } } // CLARITY is one move in two directions: the frame against its own blurred