feat(frame): read AUTO straighten from the photo, not the accelerometer

EXIF Orientation only knows 0/90/180/270, so a library still carries no
record of how the camera was held. The sensor had nothing to offer.

AUTO now measures the dominant line in the picture itself:
src/utils/horizon.ts runs a shear-projection search (coarse 1 deg over
-45..45, then a 0.5 deg refine) on a <=256px thumbnail, bails when no
line's score beats 3x the median, and returns the tilt in degrees.
App applies photoStraighten = -tilt, so preview and export share one
number exactly as the slider did.

Removes expo-sensors wiring, the horizonRoll state, effectiveStraighten
and the bubble-level overlay (autoRoll prop) from App/AdjustmentPanel/
Viewfinder. expo-sensors stays in package.json.
This commit is contained in:
2026-09-11 14:46:28 +07:00
parent d6b3ebda40
commit 1b632ba78c
4 changed files with 179 additions and 104 deletions
+15 -29
View File
@@ -89,8 +89,9 @@ interface AdjustmentPanelProps {
photoStraighten: number;
onPickRotation: (deg: 0 | 90 | 180 | 270) => void;
onStraighten: (deg: number) => void;
autoHorizon: boolean;
onAutoHorizon: (on: boolean) => void;
// FRAME-tab AUTO: ask App to read the tilt of the photo itself and set the
// STRAIGHTEN angle that cancels it. A one-shot, not a mode.
onAutoStraighten: () => void;
// FRAME tab (WALL FRAME only): hang the artwork landscape instead of the
// default portrait rotation.
wallLandscape: boolean;
@@ -122,9 +123,6 @@ interface AdjustmentPanelProps {
// FRAME → WATERMARK sub-panel is open: App needs this to arm the mark's
// drag/pinch layer on the image (the old `activeTab === 'watermark'`).
onWmPanelChange: (open: boolean) => void;
// The ROTATE strip (or its STRAIGHTEN row) is open: App needs this to keep
// the image's bubble-level line on screen only while that strip is up.
onRotatePanelChange: (open: boolean) => void;
}
interface ChipDef {
@@ -225,8 +223,7 @@ export default function AdjustmentPanel({
photoStraighten,
onPickRotation,
onStraighten,
autoHorizon,
onAutoHorizon,
onAutoStraighten,
wallLandscape,
onToggleWallLandscape,
onToggleGeotag,
@@ -241,7 +238,6 @@ export default function AdjustmentPanel({
wmRotation,
onRotateWm,
onWmPanelChange,
onRotatePanelChange,
}: AdjustmentPanelProps) {
// Double-tap on the mark in the viewfinder bumps wmEditNonce -> focus the
// input again so the text can be retyped after the keyboard was dismissed.
@@ -277,12 +273,6 @@ export default function AdjustmentPanel({
useEffect(() => {
onWmPanelChange(openGroup === 'wm');
}, [openGroup]);
// The bubble-level line on the image belongs to the ROTATE strip: it shows
// while that strip (or its STRAIGHTEN row) is open and leaves with the next
// chip, so it never lingers over the photo after the user moved on.
useEffect(() => {
onRotatePanelChange(openGroup === 'rotate' || openParam === 'straighten');
}, [openGroup, openParam]);
// CROP is a plain-frame tool: the moment the selected frame drops it the chip
// unmounts, so the ratio strip must close with it (no orphan CROP row).
useEffect(() => {
@@ -593,12 +583,12 @@ export default function AdjustmentPanel({
],
onPick: (v) => onUpdateCropRatio(v as CropRatio),
},
// ROTATE: the quarter turns, the fine STRAIGHTEN slider and AUTO horizon in
// one strip. Its RESET undoes this parameter only (turn + angle + auto).
// ROTATE: the quarter turns, the fine STRAIGHTEN slider and the AUTO read
// in one strip. Its RESET undoes this parameter only (turn + angle).
rotate: {
label: 'ROTATE', off: '0',
// 'straighten' while its slider row is open, so that chip highlights.
value: openParam === 'straighten' ? 'straighten' : autoHorizon ? 'auto' : String(photoRotation),
value: openParam === 'straighten' ? 'straighten' : String(photoRotation),
options: [
{ v: 'reset', d: 'RESET' },
{ v: 'auto', d: 'AUTO' },
@@ -612,15 +602,13 @@ export default function AdjustmentPanel({
if (v === 'reset') {
onPickRotation(0);
onStraighten(0);
onAutoHorizon(false);
} else if (v === 'straighten') {
toggleParam('straighten');
} else if (v === 'auto') {
// AUTO is "snap to the sensor horizon": the manual angle is dropped
// first, so enabling it with a hand-set offset really levels the
// photo (re-tapping with it on still switches it back off).
if (!autoHorizon) onStraighten(0);
onAutoHorizon(!autoHorizon);
// AUTO reads the photo and REPLACES the fine angle with the one
// that levels its dominant line, so pressing it twice is the same
// as pressing it once.
onAutoStraighten();
} else {
onPickRotation(Number(v) as 0 | 90 | 180 | 270);
}
@@ -671,19 +659,17 @@ export default function AdjustmentPanel({
label: wbTempLabel(),
});
// ROTATE's chip names every part of the angle it applies: the turn (dropped
// when it is 0 and AUTO is doing the work), the horizon mode, the fine angle.
// ROTATE's chip names every part of the angle it applies: the turn and the
// fine angle AUTO (or the slider) left on it.
const rotateChip = (): ChipDef => {
const parts = [
photoRotation !== 0 || !autoHorizon ? `ROTATE ${photoRotation}` : 'ROTATE',
autoHorizon ? 'AUTO' : null,
`ROTATE ${photoRotation}`,
photoStraighten ? `${photoStraighten > 0 ? '+' : ''}${photoStraighten}°` : null,
];
return {
...groupChip('rotate'),
label: parts.filter(Boolean).join(' · '),
amberValue:
openGroup !== 'rotate' && (photoRotation !== 0 || photoStraighten !== 0 || autoHorizon),
amberValue: openGroup !== 'rotate' && (photoRotation !== 0 || photoStraighten !== 0),
};
};
+19 -25
View File
@@ -33,6 +33,7 @@ import { TONE_SKSL, getToneUniforms, toneUniformArray } from '../utils/toneShade
import { CINEMA_SKSL, getCinemaUniforms, cinemaOffFlat } from '../utils/cinemaShader';
import { polaroidLayout, wallframeLayout, WALLFRAME_PNG_W, WALLFRAME_PNG_H } from '../utils/frameUtils';
import { applyPhotoRotation } from '../utils/skiaImage';
import { detectTilt } from '../utils/horizon';
interface ViewfinderProps {
mode: 'camera' | 'library';
@@ -54,9 +55,6 @@ interface ViewfinderProps {
// on top of that turn.
photoRotation?: 0 | 90 | 180 | 270;
photoStraighten?: number;
// FRAME-tab AUTO: the device's lean in degrees, null while AUTO is off. Only
// drives the level line — the photo arrives already counter-rotated.
autoRoll?: number | null;
// WATERMARK tab: double-tap ON the mark re-opens the text editor (App bumps
// a nonce so the panel puts the caret back in its input).
onWmEdit?: () => void;
@@ -164,6 +162,14 @@ export interface ViewfinderHandle {
* fixed-ratio crop is active.
*/
getCropRatioRect(): CropRect | null;
/**
* FRAME-tab AUTO straighten: tilt, in degrees, of the dominant near-
* horizontal line in the library photo — the horizon it was shot with.
* Null when the picture holds no line worth trusting, so App can leave the
* fine angle alone. Read from the quarter-turned source WITHOUT the current
* straighten, so pressing AUTO twice answers the same both times.
*/
detectHorizon(): number | null;
}
const noiseEffect = Skia.RuntimeEffect.Make(`
@@ -237,7 +243,6 @@ const Viewfinder = forwardRef<ViewfinderHandle, ViewfinderProps>(function Viewfi
wmEditing = false,
photoRotation = 0,
photoStraighten = 0,
autoRoll = null,
wmRotation = 0,
wallLandscape = false,
}: ViewfinderProps,
@@ -338,6 +343,10 @@ const Viewfinder = forwardRef<ViewfinderHandle, ViewfinderProps>(function Viewfi
() => applyPhotoRotation(loadedImage, photoRotation, photoStraighten),
[loadedImage, photoRotation, photoStraighten]
);
// Mirror of the two inputs of the above, for detectHorizon: the imperative
// handle is captured once (deps [ref]) and must read the LIVE source.
const horizonSrcRef = useRef({ image: loadedImage, quarter: photoRotation });
horizonSrcRef.current = { image: loadedImage, quarter: photoRotation };
// Wall-frame PNG artwork (project root). Loaded lazily so other frames pay
// no asset cost; source switches to '' when unselected (useImage → null).
const wallImage = useImage(selectedFrame === 'wallframe' ? require('../../wallframe.png') : '');
@@ -580,6 +589,12 @@ const Viewfinder = forwardRef<ViewfinderHandle, ViewfinderProps>(function Viewfi
getCropRatioRect() {
return cropRatioRectRef.current;
},
// AUTO straighten: the angle the picture asks for. Computed on demand
// (one <=256px thumbnail, tens of ms) — never on a slider drag.
detectHorizon() {
const src = horizonSrcRef.current;
return detectTilt(applyPhotoRotation(src.image, src.quarter, 0));
},
}),
[ref]
);
@@ -2257,24 +2272,6 @@ const Viewfinder = forwardRef<ViewfinderHandle, ViewfinderProps>(function Viewfi
const cinemaOn = cinemaEffect != null && !!cinemaParams;
const useShaderPass = !!imageFitRect && ((toneEffect != null && toneOn) || cinemaOn);
// FRAME-tab AUTO: a bubble level that follows the device's own lean. The photo
// under it is already counter-rotated by the same angle, so this line is the
// only place the raw tilt is visible. Amber once within a degree of level.
const levelOverlay = autoRoll == null ? null : (
<View pointerEvents="none" className="absolute inset-0 items-center justify-center">
<View style={{ width: '55%', transform: [{ rotate: `${autoRoll}deg` }] }}>
<View style={{ height: 4, backgroundColor: 'rgba(0,0,0,0.5)' }} />
<View
style={{
height: 2,
marginTop: -3,
backgroundColor: Math.abs(autoRoll) <= 1 ? '#f59e0b' : '#ffffff',
}}
/>
</View>
</View>
);
// RETRO INSTANT (library): the whole picture is contain-fitted into the
// card's photo window — nothing is masked or cropped, and the card is
// centered on the black background like a physical print. The window keeps
@@ -2350,7 +2347,6 @@ const Viewfinder = forwardRef<ViewfinderHandle, ViewfinderProps>(function Viewfi
onResponderMove={onLibTouchMove}
onResponderRelease={onLibTouchEnd}
/>
{levelOverlay}
</View>
);
}
@@ -2427,7 +2423,6 @@ const Viewfinder = forwardRef<ViewfinderHandle, ViewfinderProps>(function Viewfi
onResponderMove={onLibTouchMove}
onResponderRelease={onLibTouchEnd}
/>
{levelOverlay}
</View>
);
}
@@ -2570,7 +2565,6 @@ const Viewfinder = forwardRef<ViewfinderHandle, ViewfinderProps>(function Viewfi
onResponderMove={onLibTouchMove}
onResponderRelease={onLibTouchEnd}
/>
{levelOverlay}
</>
) : (
<View className="items-center justify-center p-6">
+128
View File
@@ -0,0 +1,128 @@
import { Skia, FilterMode, MipmapMode, type SkImage } from '@shopify/react-native-skia';
// FRAME-tab AUTO straighten: the angle, in degrees, of the horizon in the photo.
// Null when the picture has no line worth trusting (a portrait, a flat wall), so
// AUTO leaves the fine angle alone instead of inventing one.
//
// Deskew by projection profile, not a gradient-orientation histogram: for every
// candidate angle the edge map is sheared and projected onto the rows, and the
// angle whose projection is the sharpest single line wins. The histogram looks
// cheaper, but a real edge is a staircase one pixel tall — a 7 degree line only
// steps every 8 px — so its gradients point at 0 and 45 degrees instead of at
// the line, and every small tilt reads as zero. Shearing cancels a staircase
// instead of averaging it away.
//
// ponytail: 1 degree coarse pass, 0.5 degree refine around the winner. The
// STRAIGHTEN slider is a 0.5 degree grid, so finer is invisible. Add a third
// pass if a residual tilt ever shows up.
export function detectTilt(image: SkImage | null, maxAngle = 45): number | null {
if (!image) return null;
let iw = image.width();
let ih = image.height();
if (!(iw > 8 && ih > 8)) return null;
// Scan a thumbnail, never the 12 MP original: the profile only needs the
// gross shape of the lines, and this runs on the JS thread. Halve instead of
// resampling in one go — a one-shot 3x bilinear step samples 4 taps spread
// over 3 source pixels and leaves exactly the staircase the search undoes.
let cur = image;
while (Math.max(iw, ih) > 256) {
const w = Math.max(8, Math.round(iw / 2));
const h = Math.max(8, Math.round(ih / 2));
const surface = Skia.Surface.Make(w, h);
if (!surface) break;
surface
.getCanvas()
.drawImageRectOptions(
cur,
Skia.XYWHRect(0, 0, iw, ih),
Skia.XYWHRect(0, 0, w, h),
FilterMode.Linear,
MipmapMode.Linear
);
const next = surface.makeImageSnapshot();
if (!next) break;
cur = next;
iw = w;
ih = h;
}
const raw = cur.readPixels();
if (!raw) return null;
// Bytes for the 8-bit raster surfaces this app builds; the float branch is
// there because readPixels() reports whatever the image holds.
const unit = raw instanceof Float32Array ? 255 : 1;
const w = iw;
const h = ih;
const gray = new Float32Array(w * h);
for (let i = 0; i < w * h; i++) {
gray[i] = (0.299 * raw[i * 4] + 0.587 * raw[i * 4 + 1] + 0.114 * raw[i * 4 + 2]) * unit;
}
// The edge map, kept as a list: the angle search touches every edge once per
// candidate, so skipping the flat pixels is most of the saving.
const px = new Float64Array(w * h);
const py = new Float64Array(w * h);
const pw = new Float64Array(w * h);
let n = 0;
for (let y = 1; y < h - 1; y++) {
for (let x = 1; x < w - 1; x++) {
const i = y * w + x;
const dx = gray[i + 1] - gray[i - 1];
const dy = gray[i + w] - gray[i - w];
const mag = Math.sqrt(dx * dx + dy * dy);
if (mag < 20) continue; // flat patch: no line edge here
px[n] = x - w / 2;
py[n] = y;
pw[n] = mag;
n++;
}
}
if (n < 32) return null; // too few edges to call anything a horizon
// Score an angle by how sharply the sheared edge map projects onto the rows:
// a line sheared to its own angle lands in one bin, any other angle smears it
// over |delta| * w / 57 bins wide and the squared-sum collapses.
const limit = Math.ceil((w / 2) * Math.tan((maxAngle * Math.PI) / 180)) + 2;
const size = h + 2 * limit;
const acc = new Float64Array(size);
const score = (deg: number) => {
acc.fill(0);
const t = Math.tan((deg * Math.PI) / 180);
for (let k = 0; k < n; k++) acc[Math.round(py[k] - px[k] * t) + limit] += pw[k];
let ss = 0;
let tot = 0;
for (let i = 0; i < size; i++) {
ss += acc[i] * acc[i];
tot += acc[i];
}
return tot > 0 ? ss / (tot * tot) : 0;
};
const coarse: number[] = [];
let best = 0;
let bestScore = -1;
for (let deg = -maxAngle; deg <= maxAngle; deg++) {
const s = score(deg);
coarse.push(s);
if (s > bestScore) {
bestScore = s;
best = deg;
}
}
// A picture with no line scores about the same at every angle; only a real
// one stands clear of the middle of the field.
coarse.sort((a, b) => a - b);
if (bestScore < 3 * coarse[coarse.length >> 1]) return null;
let angle = best;
for (let deg = best - 1; deg <= best + 1; deg += 0.5) {
const s = score(deg);
if (s > bestScore) {
bestScore = s;
angle = deg;
}
}
return angle;
}