From 1b632ba78ccd259990d11390685e5b252cd2e653 Mon Sep 17 00:00:00 2001 From: locphamtran Date: Fri, 11 Sep 2026 14:46:28 +0700 Subject: [PATCH] feat(frame): read AUTO straighten from the photo, not the accelerometer EXIF Orientation only knows 0/90/180/270, so a library still carries no record of how the camera was held. The sensor had nothing to offer. AUTO now measures the dominant line in the picture itself: src/utils/horizon.ts runs a shear-projection search (coarse 1 deg over -45..45, then a 0.5 deg refine) on a <=256px thumbnail, bails when no line's score beats 3x the median, and returns the tilt in degrees. App applies photoStraighten = -tilt, so preview and export share one number exactly as the slider did. Removes expo-sensors wiring, the horizonRoll state, effectiveStraighten and the bubble-level overlay (autoRoll prop) from App/AdjustmentPanel/ Viewfinder. expo-sensors stays in package.json. --- App.tsx | 67 ++++----------- src/components/AdjustmentPanel.tsx | 44 ++++------ src/components/Viewfinder.tsx | 44 +++++----- src/utils/horizon.ts | 128 +++++++++++++++++++++++++++++ 4 files changed, 179 insertions(+), 104 deletions(-) create mode 100644 src/utils/horizon.ts diff --git a/App.tsx b/App.tsx index da18de2..01f3a71 100644 --- a/App.tsx +++ b/App.tsx @@ -6,7 +6,6 @@ import * as FileSystem from 'expo-file-system/legacy'; import * as MediaLibrary from 'expo-media-library'; import * as MediaLibraryLegacy from 'expo-media-library/legacy'; import * as Haptics from 'expo-haptics'; -import { Accelerometer } from 'expo-sensors'; import AsyncStorage from '@react-native-async-storage/async-storage'; import TopBar from './src/components/TopBar'; @@ -72,35 +71,6 @@ export default function App() { // export alike, the straighten angle applied on top of the turn. const [photoRotation, setPhotoRotation] = useState<0 | 90 | 180 | 270>(0); const [photoStraighten, setPhotoStraighten] = useState(0); - // FRAME-tab AUTO: level the photo to the horizon. The accelerometer feeds the - // very same straighten angle the hand slider does (as an offset on top of - // it), so preview and export share one number. Library only — a live camera - // frame has no horizon to bake in. - const [autoHorizon, setAutoHorizon] = useState(false); - const [horizonRoll, setHorizonRoll] = useState(0); - useEffect(() => { - if (!autoHorizon || mode !== 'library') return; - Accelerometer.setUpdateInterval(100); - const sub = Accelerometer.addListener(({ x, y }) => { - // atan2(x, y) is the lean about the screen normal: 0 upright in portrait. - // Skip a near-flat hold — x/y are noise there and no horizon is readable. - // Whole degrees only: a 12 MP re-rotation per sensor tick would burn the - // battery for a sub-pixel move. - if (Math.hypot(x, y) < 0.2) return; - const deg = (Math.atan2(x, y) * 180) / Math.PI; - // Past 45 the phone is not being held to shoot: the level line and the - // angle both stop meaning anything, so keep the last good reading. - if (Math.abs(deg) > 45) return; - setHorizonRoll(Math.round(deg)); - }); - return () => sub.remove(); - }, [autoHorizon, mode]); - // Lean the phone and everything painted on its screen leans with it, so the - // photo gains exactly the lean the STRAIGHTEN slider would have to undo: the - // measured roll is added, never negated. An emulator tilt of +10° puts the - // export 10° clockwise of the untilted one. - const effectiveStraighten = - photoStraighten + (autoHorizon && mode === 'library' ? horizonRoll : 0); useEffect(() => { const show = Keyboard.addListener('keyboardDidShow', (e) => setKeyboardHeight(e.endCoordinates.height)); const hide = Keyboard.addListener('keyboardDidHide', () => setKeyboardHeight(0)); @@ -223,9 +193,6 @@ export default function App() { const [openParamKey, setOpenParamKey] = useState(null); // FRAME → WATERMARK sub-panel is open (the rail's WATERMARK tab is gone). const [wmPanelOpen, setWmPanelOpen] = useState(false); - // FRAME -> ROTATE strip is open: gates the bubble-level line on the image. - const [rotatePanelOpen, setRotatePanelOpen] = useState(false); - const [libraryImageUri, setLibraryImageUri] = useState(null); // Session photo history (camera captures + library exports). The camera // thumbnail/preview button shows the newest entry; the swipe viewer walks the @@ -571,8 +538,6 @@ export default function App() { setWmRotation(0); setPhotoRotation(0); setPhotoStraighten(0); - setAutoHorizon(false); - setHorizonRoll(0); setWallLandscape(false); setCropRatio('none'); setCropRect(DEFAULT_CROP_RECT); @@ -593,7 +558,6 @@ export default function App() { wmRotation !== 0 || photoRotation !== 0 || photoStraighten !== 0 || - autoHorizon || wallLandscape || cropRatio !== 'none' || differs(cropRect, DEFAULT_CROP_RECT); @@ -825,6 +789,18 @@ export default function App() { setFavoriteId(id, on).catch((e) => console.warn('Failed to save favorite:', e)); }; + // FRAME-tab AUTO: the photo is levelled from its OWN pixels, not from the + // phone. A library still carries no record of how the camera was held when + // it was shot (EXIF Orientation only knows 0/90/180/270), so the sensor had + // nothing to offer; the dominant line in the picture is the only horizon + // there is. A one-shot: it replaces whatever the slider or a previous AUTO + // left, and does nothing at all when the picture has no line worth trusting. + const handleAutoStraighten = useCallback(() => { + const tilt = viewfinderRef.current?.detectHorizon?.() ?? null; + if (tilt == null) return; + setPhotoStraighten(-tilt); + }, []); + const handlePickImage = async () => { if (isProcessing) return; @@ -859,12 +835,10 @@ export default function App() { if (!result.canceled && result.assets && result.assets.length > 0) { const asset = result.assets[0]; setLibraryImageUri(asset.uri); - // A new photo starts level: the previous photo's turn, fine straighten - // and AUTO horizon must not carry over. + // A new photo starts level: the previous photo's turn and fine + // straighten must not carry over. setPhotoRotation(0); setPhotoStraighten(0); - setAutoHorizon(false); - setHorizonRoll(0); setMode('library'); // Picking from the camera is the other half of that switch (the camera // button is the way back), so the camera's edits must not land on the @@ -1126,7 +1100,7 @@ export default function App() { // save the same crop the preview showed (null = centered 1x). frameWindowZoom: viewfinderRef.current?.getFrameWindowZoom?.() ?? null, photoRotation, - photoStraighten: effectiveStraighten, + photoStraighten, } ); @@ -1304,12 +1278,7 @@ export default function App() { libraryImageUri={libraryImageUri} availableHeight={deckTop} photoRotation={photoRotation} - photoStraighten={effectiveStraighten} - autoRoll={ - autoHorizon && mode === 'library' && openTab === 'frame' && rotatePanelOpen - ? horizonRoll - : null - } + photoStraighten={photoStraighten} cameraPermissionGranted={cameraPermissionGranted} onRequestCameraPermission={requestCameraPermission} photoOutput={photoOutput} @@ -1390,8 +1359,7 @@ export default function App() { photoStraighten={photoStraighten} onPickRotation={setPhotoRotation} onStraighten={setPhotoStraighten} - autoHorizon={autoHorizon} - onAutoHorizon={setAutoHorizon} + onAutoStraighten={handleAutoStraighten} wallLandscape={wallLandscape} onToggleWallLandscape={() => setWallLandscape((v) => !v)} useGeotag={useGeotag} @@ -1420,7 +1388,6 @@ export default function App() { wmRotation={wmRotation} onRotateWm={() => setWmRotation((r) => (r + 90) % 360)} onWmPanelChange={setWmPanelOpen} - onRotatePanelChange={setRotatePanelOpen} /> )} diff --git a/src/components/AdjustmentPanel.tsx b/src/components/AdjustmentPanel.tsx index 56f662e..350717d 100644 --- a/src/components/AdjustmentPanel.tsx +++ b/src/components/AdjustmentPanel.tsx @@ -89,8 +89,9 @@ interface AdjustmentPanelProps { photoStraighten: number; onPickRotation: (deg: 0 | 90 | 180 | 270) => void; onStraighten: (deg: number) => void; - autoHorizon: boolean; - onAutoHorizon: (on: boolean) => void; + // FRAME-tab AUTO: ask App to read the tilt of the photo itself and set the + // STRAIGHTEN angle that cancels it. A one-shot, not a mode. + onAutoStraighten: () => void; // FRAME tab (WALL FRAME only): hang the artwork landscape instead of the // default portrait rotation. wallLandscape: boolean; @@ -122,9 +123,6 @@ interface AdjustmentPanelProps { // FRAME → WATERMARK sub-panel is open: App needs this to arm the mark's // drag/pinch layer on the image (the old `activeTab === 'watermark'`). onWmPanelChange: (open: boolean) => void; - // The ROTATE strip (or its STRAIGHTEN row) is open: App needs this to keep - // the image's bubble-level line on screen only while that strip is up. - onRotatePanelChange: (open: boolean) => void; } interface ChipDef { @@ -225,8 +223,7 @@ export default function AdjustmentPanel({ photoStraighten, onPickRotation, onStraighten, - autoHorizon, - onAutoHorizon, + onAutoStraighten, wallLandscape, onToggleWallLandscape, onToggleGeotag, @@ -241,7 +238,6 @@ export default function AdjustmentPanel({ wmRotation, onRotateWm, onWmPanelChange, - onRotatePanelChange, }: AdjustmentPanelProps) { // Double-tap on the mark in the viewfinder bumps wmEditNonce -> focus the // input again so the text can be retyped after the keyboard was dismissed. @@ -277,12 +273,6 @@ export default function AdjustmentPanel({ useEffect(() => { onWmPanelChange(openGroup === 'wm'); }, [openGroup]); - // The bubble-level line on the image belongs to the ROTATE strip: it shows - // while that strip (or its STRAIGHTEN row) is open and leaves with the next - // chip, so it never lingers over the photo after the user moved on. - useEffect(() => { - onRotatePanelChange(openGroup === 'rotate' || openParam === 'straighten'); - }, [openGroup, openParam]); // CROP is a plain-frame tool: the moment the selected frame drops it the chip // unmounts, so the ratio strip must close with it (no orphan CROP row). useEffect(() => { @@ -593,12 +583,12 @@ export default function AdjustmentPanel({ ], onPick: (v) => onUpdateCropRatio(v as CropRatio), }, - // ROTATE: the quarter turns, the fine STRAIGHTEN slider and AUTO horizon in - // one strip. Its RESET undoes this parameter only (turn + angle + auto). + // ROTATE: the quarter turns, the fine STRAIGHTEN slider and the AUTO read + // in one strip. Its RESET undoes this parameter only (turn + angle). rotate: { label: 'ROTATE', off: '0', // 'straighten' while its slider row is open, so that chip highlights. - value: openParam === 'straighten' ? 'straighten' : autoHorizon ? 'auto' : String(photoRotation), + value: openParam === 'straighten' ? 'straighten' : String(photoRotation), options: [ { v: 'reset', d: 'RESET' }, { v: 'auto', d: 'AUTO' }, @@ -612,15 +602,13 @@ export default function AdjustmentPanel({ if (v === 'reset') { onPickRotation(0); onStraighten(0); - onAutoHorizon(false); } else if (v === 'straighten') { toggleParam('straighten'); } else if (v === 'auto') { - // AUTO is "snap to the sensor horizon": the manual angle is dropped - // first, so enabling it with a hand-set offset really levels the - // photo (re-tapping with it on still switches it back off). - if (!autoHorizon) onStraighten(0); - onAutoHorizon(!autoHorizon); + // AUTO reads the photo and REPLACES the fine angle with the one + // that levels its dominant line, so pressing it twice is the same + // as pressing it once. + onAutoStraighten(); } else { onPickRotation(Number(v) as 0 | 90 | 180 | 270); } @@ -671,19 +659,17 @@ export default function AdjustmentPanel({ label: wbTempLabel(), }); - // ROTATE's chip names every part of the angle it applies: the turn (dropped - // when it is 0 and AUTO is doing the work), the horizon mode, the fine angle. + // ROTATE's chip names every part of the angle it applies: the turn and the + // fine angle AUTO (or the slider) left on it. const rotateChip = (): ChipDef => { const parts = [ - photoRotation !== 0 || !autoHorizon ? `ROTATE ${photoRotation}` : 'ROTATE', - autoHorizon ? 'AUTO' : null, + `ROTATE ${photoRotation}`, photoStraighten ? `${photoStraighten > 0 ? '+' : ''}${photoStraighten}°` : null, ]; return { ...groupChip('rotate'), label: parts.filter(Boolean).join(' · '), - amberValue: - openGroup !== 'rotate' && (photoRotation !== 0 || photoStraighten !== 0 || autoHorizon), + amberValue: openGroup !== 'rotate' && (photoRotation !== 0 || photoStraighten !== 0), }; }; diff --git a/src/components/Viewfinder.tsx b/src/components/Viewfinder.tsx index 1717b40..bda35bc 100644 --- a/src/components/Viewfinder.tsx +++ b/src/components/Viewfinder.tsx @@ -33,6 +33,7 @@ import { TONE_SKSL, getToneUniforms, toneUniformArray } from '../utils/toneShade import { CINEMA_SKSL, getCinemaUniforms, cinemaOffFlat } from '../utils/cinemaShader'; import { polaroidLayout, wallframeLayout, WALLFRAME_PNG_W, WALLFRAME_PNG_H } from '../utils/frameUtils'; import { applyPhotoRotation } from '../utils/skiaImage'; +import { detectTilt } from '../utils/horizon'; interface ViewfinderProps { mode: 'camera' | 'library'; @@ -54,9 +55,6 @@ interface ViewfinderProps { // on top of that turn. photoRotation?: 0 | 90 | 180 | 270; photoStraighten?: number; - // FRAME-tab AUTO: the device's lean in degrees, null while AUTO is off. Only - // drives the level line — the photo arrives already counter-rotated. - autoRoll?: number | null; // WATERMARK tab: double-tap ON the mark re-opens the text editor (App bumps // a nonce so the panel puts the caret back in its input). onWmEdit?: () => void; @@ -164,6 +162,14 @@ export interface ViewfinderHandle { * fixed-ratio crop is active. */ getCropRatioRect(): CropRect | null; + /** + * FRAME-tab AUTO straighten: tilt, in degrees, of the dominant near- + * horizontal line in the library photo — the horizon it was shot with. + * Null when the picture holds no line worth trusting, so App can leave the + * fine angle alone. Read from the quarter-turned source WITHOUT the current + * straighten, so pressing AUTO twice answers the same both times. + */ + detectHorizon(): number | null; } const noiseEffect = Skia.RuntimeEffect.Make(` @@ -237,7 +243,6 @@ const Viewfinder = forwardRef(function Viewfi wmEditing = false, photoRotation = 0, photoStraighten = 0, - autoRoll = null, wmRotation = 0, wallLandscape = false, }: ViewfinderProps, @@ -338,6 +343,10 @@ const Viewfinder = forwardRef(function Viewfi () => applyPhotoRotation(loadedImage, photoRotation, photoStraighten), [loadedImage, photoRotation, photoStraighten] ); + // Mirror of the two inputs of the above, for detectHorizon: the imperative + // handle is captured once (deps [ref]) and must read the LIVE source. + const horizonSrcRef = useRef({ image: loadedImage, quarter: photoRotation }); + horizonSrcRef.current = { image: loadedImage, quarter: photoRotation }; // Wall-frame PNG artwork (project root). Loaded lazily so other frames pay // no asset cost; source switches to '' when unselected (useImage → null). const wallImage = useImage(selectedFrame === 'wallframe' ? require('../../wallframe.png') : ''); @@ -580,6 +589,12 @@ const Viewfinder = forwardRef(function Viewfi getCropRatioRect() { return cropRatioRectRef.current; }, + // AUTO straighten: the angle the picture asks for. Computed on demand + // (one <=256px thumbnail, tens of ms) — never on a slider drag. + detectHorizon() { + const src = horizonSrcRef.current; + return detectTilt(applyPhotoRotation(src.image, src.quarter, 0)); + }, }), [ref] ); @@ -2257,24 +2272,6 @@ const Viewfinder = forwardRef(function Viewfi const cinemaOn = cinemaEffect != null && !!cinemaParams; const useShaderPass = !!imageFitRect && ((toneEffect != null && toneOn) || cinemaOn); - // FRAME-tab AUTO: a bubble level that follows the device's own lean. The photo - // under it is already counter-rotated by the same angle, so this line is the - // only place the raw tilt is visible. Amber once within a degree of level. - const levelOverlay = autoRoll == null ? null : ( - - - - - - - ); - // RETRO INSTANT (library): the whole picture is contain-fitted into the // card's photo window — nothing is masked or cropped, and the card is // centered on the black background like a physical print. The window keeps @@ -2350,7 +2347,6 @@ const Viewfinder = forwardRef(function Viewfi onResponderMove={onLibTouchMove} onResponderRelease={onLibTouchEnd} /> - {levelOverlay} ); } @@ -2427,7 +2423,6 @@ const Viewfinder = forwardRef(function Viewfi onResponderMove={onLibTouchMove} onResponderRelease={onLibTouchEnd} /> - {levelOverlay} ); } @@ -2570,7 +2565,6 @@ const Viewfinder = forwardRef(function Viewfi onResponderMove={onLibTouchMove} onResponderRelease={onLibTouchEnd} /> - {levelOverlay} ) : ( diff --git a/src/utils/horizon.ts b/src/utils/horizon.ts new file mode 100644 index 0000000..628178b --- /dev/null +++ b/src/utils/horizon.ts @@ -0,0 +1,128 @@ +import { Skia, FilterMode, MipmapMode, type SkImage } from '@shopify/react-native-skia'; + +// FRAME-tab AUTO straighten: the angle, in degrees, of the horizon in the photo. +// Null when the picture has no line worth trusting (a portrait, a flat wall), so +// AUTO leaves the fine angle alone instead of inventing one. +// +// Deskew by projection profile, not a gradient-orientation histogram: for every +// candidate angle the edge map is sheared and projected onto the rows, and the +// angle whose projection is the sharpest single line wins. The histogram looks +// cheaper, but a real edge is a staircase one pixel tall — a 7 degree line only +// steps every 8 px — so its gradients point at 0 and 45 degrees instead of at +// the line, and every small tilt reads as zero. Shearing cancels a staircase +// instead of averaging it away. +// +// ponytail: 1 degree coarse pass, 0.5 degree refine around the winner. The +// STRAIGHTEN slider is a 0.5 degree grid, so finer is invisible. Add a third +// pass if a residual tilt ever shows up. +export function detectTilt(image: SkImage | null, maxAngle = 45): number | null { + if (!image) return null; + let iw = image.width(); + let ih = image.height(); + if (!(iw > 8 && ih > 8)) return null; + + // Scan a thumbnail, never the 12 MP original: the profile only needs the + // gross shape of the lines, and this runs on the JS thread. Halve instead of + // resampling in one go — a one-shot 3x bilinear step samples 4 taps spread + // over 3 source pixels and leaves exactly the staircase the search undoes. + let cur = image; + while (Math.max(iw, ih) > 256) { + const w = Math.max(8, Math.round(iw / 2)); + const h = Math.max(8, Math.round(ih / 2)); + const surface = Skia.Surface.Make(w, h); + if (!surface) break; + surface + .getCanvas() + .drawImageRectOptions( + cur, + Skia.XYWHRect(0, 0, iw, ih), + Skia.XYWHRect(0, 0, w, h), + FilterMode.Linear, + MipmapMode.Linear + ); + const next = surface.makeImageSnapshot(); + if (!next) break; + cur = next; + iw = w; + ih = h; + } + + const raw = cur.readPixels(); + if (!raw) return null; + // Bytes for the 8-bit raster surfaces this app builds; the float branch is + // there because readPixels() reports whatever the image holds. + const unit = raw instanceof Float32Array ? 255 : 1; + const w = iw; + const h = ih; + + const gray = new Float32Array(w * h); + for (let i = 0; i < w * h; i++) { + gray[i] = (0.299 * raw[i * 4] + 0.587 * raw[i * 4 + 1] + 0.114 * raw[i * 4 + 2]) * unit; + } + + // The edge map, kept as a list: the angle search touches every edge once per + // candidate, so skipping the flat pixels is most of the saving. + const px = new Float64Array(w * h); + const py = new Float64Array(w * h); + const pw = new Float64Array(w * h); + let n = 0; + for (let y = 1; y < h - 1; y++) { + for (let x = 1; x < w - 1; x++) { + const i = y * w + x; + const dx = gray[i + 1] - gray[i - 1]; + const dy = gray[i + w] - gray[i - w]; + const mag = Math.sqrt(dx * dx + dy * dy); + if (mag < 20) continue; // flat patch: no line edge here + px[n] = x - w / 2; + py[n] = y; + pw[n] = mag; + n++; + } + } + if (n < 32) return null; // too few edges to call anything a horizon + + // Score an angle by how sharply the sheared edge map projects onto the rows: + // a line sheared to its own angle lands in one bin, any other angle smears it + // over |delta| * w / 57 bins wide and the squared-sum collapses. + const limit = Math.ceil((w / 2) * Math.tan((maxAngle * Math.PI) / 180)) + 2; + const size = h + 2 * limit; + const acc = new Float64Array(size); + const score = (deg: number) => { + acc.fill(0); + const t = Math.tan((deg * Math.PI) / 180); + for (let k = 0; k < n; k++) acc[Math.round(py[k] - px[k] * t) + limit] += pw[k]; + let ss = 0; + let tot = 0; + for (let i = 0; i < size; i++) { + ss += acc[i] * acc[i]; + tot += acc[i]; + } + return tot > 0 ? ss / (tot * tot) : 0; + }; + + const coarse: number[] = []; + let best = 0; + let bestScore = -1; + for (let deg = -maxAngle; deg <= maxAngle; deg++) { + const s = score(deg); + coarse.push(s); + if (s > bestScore) { + bestScore = s; + best = deg; + } + } + // A picture with no line scores about the same at every angle; only a real + // one stands clear of the middle of the field. + coarse.sort((a, b) => a - b); + if (bestScore < 3 * coarse[coarse.length >> 1]) return null; + + let angle = best; + for (let deg = best - 1; deg <= best + 1; deg += 0.5) { + const s = score(deg); + if (s > bestScore) { + bestScore = s; + angle = deg; + } + } + return angle; +}