From eafb3a4937aa948dcaa7ff455b11f548bcf4b1c4 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Fri, 1 May 2026 17:44:29 -0700 Subject: [PATCH 01/14] feat: added audio track spectrogram visualization feature --- package-lock.json | 11 +- package.json | 1 + src/components/MainContent.tsx | 34 ++- src/components/piano-roll/PianoGrid.tsx | 30 +- src/components/piano-roll/PianoRoll.css | 30 ++ src/components/piano-roll/PianoRoll.tsx | 33 ++- .../piano-roll/PianoRollContent.tsx | 41 ++- .../piano-roll/PianoRollToolbar.tsx | 185 ++++++++----- .../piano-roll/SpectrogramCanvas.tsx | 256 ++++++++++++++++++ src/components/track/Region.css | 21 ++ src/components/track/RegionItem.tsx | 24 ++ src/components/track/TrackGridItem.tsx | 5 + src/components/track/TrackGridPanel.tsx | 3 + src/stores/projectStore.ts | 14 +- src/workers/spectrogramWorker.ts | 96 +++++++ 15 files changed, 696 insertions(+), 88 deletions(-) create mode 100644 src/components/piano-roll/SpectrogramCanvas.tsx create mode 100644 src/workers/spectrogramWorker.ts diff --git a/package-lock.json b/package-lock.json index 5454528..70ced97 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,16 @@ { "name": "K.G.Studio", - "version": "0.10.0-build.20260411", + "version": "0.12.0-build.20260430", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "K.G.Studio", - "version": "0.10.0-build.20260411", + "version": "0.12.0-build.20260430", "dependencies": { "@breezystack/lamejs": "^1.2.7", "class-transformer": "^0.5.1", + "fft.js": "^4.0.4", "idb": "^8.0.3", "jszip": "^3.10.1", "openai": "^6.33.0", @@ -4691,6 +4692,12 @@ "dev": true, "license": "MIT" }, + "node_modules/fft.js": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/fft.js/-/fft.js-4.0.4.tgz", + "integrity": "sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==", + "license": "MIT" + }, "node_modules/figures": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/figures/-/figures-5.0.0.tgz", diff --git a/package.json b/package.json index 90d7bfd..05bdb79 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "dependencies": { "@breezystack/lamejs": "^1.2.7", "class-transformer": "^0.5.1", + "fft.js": "^4.0.4", "idb": "^8.0.3", "jszip": "^3.10.1", "openai": "^6.33.0", diff --git a/src/components/MainContent.tsx b/src/components/MainContent.tsx index d5cce65..444155a 100644 --- a/src/components/MainContent.tsx +++ b/src/components/MainContent.tsx @@ -41,9 +41,12 @@ const MainContent: React.FC = ({ activeRegionId, setShowPianoRoll, setActiveRegionId, + pianoRollMode, + openSpectrogramViewer, addTrack, addAudioTrack, projectName, + savedProjectName, } = useProjectStore(); // State to store regions @@ -533,6 +536,12 @@ const MainContent: React.FC = ({ setShowPianoRoll(true); }; + // Handle spectrogram viewer open + const handleOpenSpectrogram = (regionId: string) => { + handleRegionClick(regionId); + openSpectrogramViewer(regionId); + }; + // Handle piano roll close const handlePianoRollClose = () => { setShowPianoRoll(false); @@ -824,16 +833,39 @@ const MainContent: React.FC = ({ onRegionUpdated={handleRegionUpdated} onRegionClick={handleRegionClick} onOpenPianoRoll={handleOpenPianoRoll} + onOpenSpectrogram={handleOpenSpectrogram} onExternalDropComplete={handleExternalDropComplete} /> - {/* Piano Roll - render using portal */} + {/* Piano Roll / Spectrogram Viewer - render using portal */} {showPianoRoll && createPortal( { + for (const track of tracks) { + const region = track.getRegions().find(r => r.getId() === activeRegionId); + if (region && region.getCurrentType() === 'KGAudioRegion') { + return region as unknown as KGAudioRegion; + } + } + return undefined; + })() + : undefined} + trackId={pianoRollMode === 'spectrogram' && activeRegionId + ? (() => { + for (const track of tracks) { + const region = track.getRegions().find(r => r.getId() === activeRegionId); + if (region) return track.getId().toString(); + } + return undefined; + })() + : undefined} + projectName={savedProjectName} />, document.body )} diff --git a/src/components/piano-roll/PianoGrid.tsx b/src/components/piano-roll/PianoGrid.tsx index 14e833c..b21d2c3 100644 --- a/src/components/piano-roll/PianoGrid.tsx +++ b/src/components/piano-roll/PianoGrid.tsx @@ -6,6 +6,8 @@ import { isModifierKeyPressed } from '../../util/osUtil'; import { generatePianoGridBackground, getMatchingChordsForPitch } from '../../util/scaleUtil'; import type { KeySignature } from '../../core/KGProject'; import { KGPianoRollState } from '../../core/state/KGPianoRollState'; +import SpectrogramCanvas from './SpectrogramCanvas'; +import type { KGAudioRegion } from '../../core/region/KGAudioRegion'; interface PianoGridProps { gridRef: MutableRefObject; @@ -24,6 +26,12 @@ interface PianoGridProps { selectedMode: string; keySignature: KeySignature; chordGuide: string; + audioRegion?: KGAudioRegion; + trackId?: string; + projectName?: string; + bpm?: number; + spectrogramThresholdDb?: number; + spectrogramPower?: number; } interface CursorPosition { @@ -44,7 +52,13 @@ const PianoGrid: React.FC = ({ regionStartBeat = 0, selectedMode, keySignature, - chordGuide + chordGuide, + audioRegion, + trackId, + projectName, + bpm = 120, + spectrogramThresholdDb = -25, + spectrogramPower = 0.5, }) => { const [cursorPosition, setCursorPosition] = useState(null); const [isModifierPressed, setIsModifierPressed] = useState(false); @@ -209,13 +223,25 @@ const PianoGrid: React.FC = ({
onMouseDown(e)} onMouseMove={handleMouseMove} onMouseLeave={handleMouseLeave} > + {/* Spectrogram layer — rendered at z-index 0, behind all highlights and notes */} + {audioRegion && trackId && projectName && ( + + )} + {/* Cursor Highlights */} {cursorPosition && ( <> diff --git a/src/components/piano-roll/PianoRoll.css b/src/components/piano-roll/PianoRoll.css index f6d9536..d4929b7 100644 --- a/src/components/piano-roll/PianoRoll.css +++ b/src/components/piano-roll/PianoRoll.css @@ -96,6 +96,36 @@ left: 0; } +/* Spectrogram toolbar controls */ +.spectrogram-toolbar-controls { + display: flex; + align-items: center; + gap: 6px; +} + +.spectrogram-control-label { + font-size: 10px; + color: #aaa; + white-space: nowrap; +} + +.spectrogram-threshold-slider { + /* Match the visual width of the Qua. Pos. / Qua. Len. buttons (~80px) */ + width: 80px; + height: 4px; + accent-color: #7a9ccf; + cursor: pointer; + margin: 0; +} + +.spectrogram-threshold-value { + font-size: 10px; + color: #e0e0e0; + min-width: 44px; + text-align: right; + white-space: nowrap; +} + .piano-roll-title { flex: 1; text-align: center; diff --git a/src/components/piano-roll/PianoRoll.tsx b/src/components/piano-roll/PianoRoll.tsx index 2fe27a5..990aed1 100644 --- a/src/components/piano-roll/PianoRoll.tsx +++ b/src/components/piano-roll/PianoRoll.tsx @@ -4,6 +4,7 @@ import type { MouseEvent } from 'react'; import { useProjectStore } from '../../stores/projectStore'; import { FaGripLines } from 'react-icons/fa'; import { KGMidiRegion } from '../../core/region/KGMidiRegion'; +import type { KGAudioRegion } from '../../core/region/KGAudioRegion'; import { DEBUG_MODE, PIANO_ROLL_CONSTANTS } from '../../constants'; import PianoRollHeader from './PianoRollHeader'; import PianoRollToolbar from './PianoRollToolbar'; @@ -22,18 +23,31 @@ interface PianoRollProps { regionId: string | null; initialPosition?: { x: number; y: number }; initialSize?: { width: number; height: number }; + mode?: 'midi-edit' | 'spectrogram'; + audioRegion?: KGAudioRegion; + trackId?: string; + projectName?: string; } const PianoRoll: React.FC = ({ onClose, regionId, initialPosition, - initialSize + initialSize, + mode = 'midi-edit', + audioRegion, + trackId, + projectName, }) => { - const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled } = useProjectStore(); + const isSpectrogram = mode === 'spectrogram'; + const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled, bpm } = useProjectStore(); // Tool state for piano roll const [activeTool, setActiveTool] = useState<'pointer' | 'pencil'>('pointer'); + + // Spectrogram controls (only used in spectrogram mode) + const [spectrogramThresholdDb, setSpectrogramThresholdDb] = useState(-25); + const [spectrogramPower, setSpectrogramPower] = useState(0.5); // Quantization state const [quantPosition, setQuantPosition] = useState('1/8'); @@ -855,6 +869,7 @@ const PianoRoll: React.FC = ({ // Get the title for the piano roll based on the active region const getPianoRollTitle = () => { + if (isSpectrogram) return audioRegion ? `SPECTROGRAM — ${audioRegion.getName()}` : 'SPECTROGRAM'; if (!activeRegion) return "EDIT NOTE CLIP"; // Calculate the bar and beat position of the region @@ -901,8 +916,13 @@ const PianoRoll: React.FC = ({ chordGuide={chordGuide} onChordGuideChange={handleChordGuideSelect} blinkButton={blinkButton} + mode={mode} + thresholdDb={spectrogramThresholdDb} + onThresholdChange={setSpectrogramThresholdDb} + power={spectrogramPower} + onPowerChange={setSpectrogramPower} /> - + = ({ selectedMode={selectedMode} keySignature={keySignature} chordGuide={chordGuide} + mode={mode} + audioRegion={audioRegion} + trackId={trackId} + projectName={projectName} + bpm={bpm} + spectrogramThresholdDb={spectrogramThresholdDb} + spectrogramPower={spectrogramPower} />
; @@ -26,6 +27,13 @@ interface PianoRollContentProps { selectedMode: string; keySignature: KeySignature; chordGuide: string; + mode?: 'midi-edit' | 'spectrogram'; + audioRegion?: KGAudioRegion; + trackId?: string; + projectName?: string; + bpm?: number; + spectrogramThresholdDb?: number; + spectrogramPower?: number; } const PianoRollContent: React.FC = ({ @@ -40,8 +48,16 @@ const PianoRollContent: React.FC = ({ onSetDeleteNotesTrigger, selectedMode, keySignature, - chordGuide + chordGuide, + mode = 'midi-edit', + audioRegion, + trackId, + projectName, + bpm = 120, + spectrogramThresholdDb = -25, + spectrogramPower = 0.5, }) => { + const isSpectrogram = mode === 'spectrogram'; // Get KGCore instance const core = KGCore.instance(); @@ -105,9 +121,8 @@ const PianoRollContent: React.FC = ({ // Combined click handler for both pointer and pencil modes const handleCombinedClick = (e: React.MouseEvent) => { - // Handle selection click (pointer mode) + if (isSpectrogram) return; handleBackgroundClick(e); - // Handle pencil mode note creation handleGridClick(e); }; @@ -144,7 +159,7 @@ const PianoRollContent: React.FC = ({ // Memoize the notes rendering to prevent unnecessary recalculations const memoizedNotes = useMemo(() => { - if (!activeRegion) return null; + if (isSpectrogram || !activeRegion) return null; if (DEBUG_MODE.PIANO_ROLL) { console.log(`Rendering notes for region: ${activeRegion.getId()}`); @@ -251,18 +266,24 @@ const PianoRollContent: React.FC = ({ {} : handleGridDoubleClick} + onClick={isSpectrogram ? () => {} : handleCombinedClick} + onMouseDown={isSpectrogram ? () => {} : handleBackgroundMouseDown} + isBoxSelecting={isSpectrogram ? false : isBoxSelectingRef.current} + selectionBox={isSpectrogram ? { startX: 0, startY: 0, endX: 0, endY: 0 } : selectionBoxRef.current} regionStartBeat={activeRegion?.getStartFromBeat() || 0} selectedMode={selectedMode} keySignature={keySignature} chordGuide={chordGuide} + audioRegion={audioRegion} + trackId={trackId} + projectName={projectName} + bpm={bpm} + spectrogramThresholdDb={spectrogramThresholdDb} + spectrogramPower={spectrogramPower} > {memoizedNotes} - {recordingNoteOverlays} + {!isSpectrogram && recordingNoteOverlays}
diff --git a/src/components/piano-roll/PianoRollToolbar.tsx b/src/components/piano-roll/PianoRollToolbar.tsx index ed12ea0..4df21b6 100644 --- a/src/components/piano-roll/PianoRollToolbar.tsx +++ b/src/components/piano-roll/PianoRollToolbar.tsx @@ -4,6 +4,13 @@ import { KGDropdown } from '../common'; import { KGPianoRollState } from '../../core/state/KGPianoRollState'; import { KGCore } from '../../core/KGCore'; +const POWER_OPTIONS = [ + { label: 'Linear', value: '1.0' }, + { label: '√ (default)', value: '0.5' }, + { label: 'Mild', value: '0.4' }, + { label: 'Strong', value: '0.3' }, +]; + interface PianoRollToolbarProps { activeTool: 'pointer' | 'pencil'; onToolSelect: (tool: 'pointer' | 'pencil') => void; @@ -17,6 +24,11 @@ interface PianoRollToolbarProps { chordGuide: string; onChordGuideChange: (value: string) => void; blinkButton?: string | null; + mode?: 'midi-edit' | 'spectrogram'; + thresholdDb?: number; + onThresholdChange?: (db: number) => void; + power?: number; + onPowerChange?: (power: number) => void; } const PianoRollToolbar: React.FC = ({ @@ -31,82 +43,117 @@ const PianoRollToolbar: React.FC = ({ onModeChange, chordGuide, onChordGuideChange, - blinkButton = null + blinkButton = null, + mode = 'midi-edit', + thresholdDb = -25, + onThresholdChange, + power = 0.5, + onPowerChange, }) => { + const isSpectrogram = mode === 'spectrogram'; + return (
-
- {/* Left section with mode and chord guide dropdowns */} - ({ label: data.name, value: id }))} - value={selectedMode} - onChange={(value) => onModeChange(value)} - label="Mode" - buttonClassName="mode-dropdown" - showValueAsLabel={true} - /> - onChordGuideChange(value)} - label="Chord" - buttonClassName="chord-guide-dropdown" - showValueAsLabel={true} - /> -
- -
- {/* Center section with pointer and pencil tools */} - - -
- + {!isSpectrogram && ( +
+ ({ label: data.name, value: id }))} + value={selectedMode} + onChange={(value) => onModeChange(value)} + label="Mode" + buttonClassName="mode-dropdown" + showValueAsLabel={true} + /> + onChordGuideChange(value)} + label="Chord" + buttonClassName="chord-guide-dropdown" + showValueAsLabel={true} + /> +
+ )} + + {!isSpectrogram && ( +
+ + +
+ )} +
- {/* Right section with quantization options */} - onSnappingSelect(value)} - label="Snap" - buttonClassName="snapping" - showValueAsLabel={true} - /> + {!isSpectrogram && ( + <> + onSnappingSelect(value)} + label="Snap" + buttonClassName="snapping" + showValueAsLabel={true} + /> + onQuantSelect('position', value)} + label="Qua. Pos." + buttonClassName={`quant-position ${blinkButton === 'quant-position' ? 'button-blink' : ''}`} + /> + onQuantSelect('length', value)} + label="Qua. Len." + buttonClassName={`quant-length ${blinkButton === 'quant-length' ? 'button-blink' : ''}`} + /> + + )} - onQuantSelect('position', value)} - label="Qua. Pos." - buttonClassName={`quant-position ${blinkButton === 'quant-position' ? 'button-blink' : ''}`} - /> - - onQuantSelect('length', value)} - label="Qua. Len." - buttonClassName={`quant-length ${blinkButton === 'quant-length' ? 'button-blink' : ''}`} - /> + {isSpectrogram && ( +
+ Floor + onThresholdChange?.(parseInt(e.target.value))} + title={`Noise floor: ${thresholdDb} dB`} + /> + {thresholdDb} dB + onPowerChange?.(parseFloat(v))} + label="Curve" + buttonClassName="curve-dropdown" + showValueAsLabel={true} + /> +
+ )}
); }; -export default PianoRollToolbar; \ No newline at end of file +export default PianoRollToolbar; diff --git a/src/components/piano-roll/SpectrogramCanvas.tsx b/src/components/piano-roll/SpectrogramCanvas.tsx new file mode 100644 index 0000000..6b4b90f --- /dev/null +++ b/src/components/piano-roll/SpectrogramCanvas.tsx @@ -0,0 +1,256 @@ +import React, { useCallback, useEffect, useRef, useState } from 'react'; +import * as Tone from 'tone'; +import { KGAudioRegion } from '../../core/region/KGAudioRegion'; +import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface'; +import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage'; +import type { SpectrogramRequest, SpectrogramResult } from '../../workers/spectrogramWorker'; + +interface SpectrogramCanvasProps { + audioRegion: KGAudioRegion; + trackId: string; + projectName: string; + bpm: number; + thresholdDb: number; + power: number; +} + +const PITCH_BINS = 128; +const HOP_SIZE = 1024; + +// Piecewise-linear RGB colormap: black → dark blue → blue → purple → red → orange → yellow +// Hue rotates 240°→300°→0°→60°, bypassing green entirely. +const COLORMAP_STOPS: Array<[number, [number, number, number]]> = [ + [0.00, [ 0, 0, 0]], + [0.15, [ 0, 0, 180]], + [0.35, [ 0, 60, 255]], + [0.55, [180, 0, 120]], + [0.70, [255, 0, 0]], + [0.85, [255, 140, 0]], + [1.00, [255, 255, 0]], +]; + +function hotColormap(v: number): [number, number, number] { + v = Math.max(0, Math.min(1, v)); + for (let i = 0; i < COLORMAP_STOPS.length - 1; i++) { + const [t0, c0] = COLORMAP_STOPS[i]; + const [t1, c1] = COLORMAP_STOPS[i + 1]; + if (v <= t1) { + const t = (v - t0) / (t1 - t0); + return [ + Math.round(c0[0] + t * (c1[0] - c0[0])), + Math.round(c0[1] + t * (c1[1] - c0[1])), + Math.round(c0[2] + t * (c1[2] - c0[2])), + ]; + } + } + return COLORMAP_STOPS[COLORMAP_STOPS.length - 1][1]; +} + +const SpectrogramCanvas: React.FC = ({ + audioRegion, + trackId, + projectName, + bpm, + thresholdDb, + power, +}) => { + const canvasRef = useRef(null); + const [loading, setLoading] = useState(true); + const workerRef = useRef(null); + + // Cache the raw linear result so threshold/power changes re-render without re-running FFT + const rawResultRef = useRef(null); + const sampleRateRef = useRef(44100); + const regionDurationRef = useRef(0); + + const renderSpectrogram = useCallback(( + result: SpectrogramResult, + sampleRate: number, + regionDurationSeconds: number, + thresholdDb: number, + power: number, + ) => { + const canvas = canvasRef.current; + if (!canvas) return; + const ctx = canvas.getContext('2d'); + if (!ctx) return; + + const beatWidth = + parseInt(getComputedStyle(document.documentElement).getPropertyValue('--region-grid-beat-width')) || 40; + const noteHeight = + parseInt(getComputedStyle(document.documentElement).getPropertyValue('--region-piano-key-height')) || 20; + + const totalBeats = (regionDurationSeconds * bpm) / 60; + const canvasWidth = Math.ceil(totalBeats * beatWidth); + const canvasHeight = PITCH_BINS * noteHeight; + + // Convert dB threshold to linear: values below this → black + const linearThreshold = Math.pow(10, thresholdDb / 20); + + // 1. Paint at natural STFT resolution onto an offscreen canvas (timeSteps × 128). + // Row i = pitchIndex i = pitch (107 − i). Apply threshold then power curve. + const offscreen = document.createElement('canvas'); + offscreen.width = result.timeSteps; + offscreen.height = PITCH_BINS; + const offCtx = offscreen.getContext('2d'); + if (!offCtx) return; + + const imgData = offCtx.createImageData(result.timeSteps, PITCH_BINS); + const pixels = imgData.data; + + for (let row = 0; row < PITCH_BINS; row++) { + const pitch = 107 - row; + for (let col = 0; col < result.timeSteps; col++) { + const idx = (row * result.timeSteps + col) * 4; + pixels[idx + 3] = 255; // always opaque + + if (pitch < 12 || pitch > 107) continue; // outside range → black + + const raw = result.data[col * PITCH_BINS + pitch]; + + // Hard threshold: values below noise floor → 0 (black) + // Re-scale surviving range to [0,1] then apply power curve + const gated = raw < linearThreshold + ? 0 + : Math.pow((raw - linearThreshold) / (1 - linearThreshold), power); + + if (gated <= 0) continue; // stays black + + const [r, g, b] = hotColormap(gated); + pixels[idx] = r; + pixels[idx + 1] = g; + pixels[idx + 2] = b; + } + } + + offCtx.putImageData(imgData, 0, 0); + + // 2. Stretch onto the full canvas — browser bilinear filter smooths between bins. + canvas.width = canvasWidth; + canvas.height = canvasHeight; + ctx.imageSmoothingEnabled = true; + ctx.imageSmoothingQuality = 'high'; + ctx.drawImage(offscreen, 0, 0, canvasWidth, canvasHeight); + }, [bpm]); + + // Re-render without re-running the worker when threshold or power changes + useEffect(() => { + if (rawResultRef.current) { + renderSpectrogram( + rawResultRef.current, + sampleRateRef.current, + regionDurationRef.current, + thresholdDb, + power, + ); + } + }, [thresholdDb, power, renderSpectrogram]); + + // Load audio + run worker when the audio region itself changes + useEffect(() => { + let cancelled = false; + + const compute = async () => { + setLoading(true); + workerRef.current?.terminate(); + workerRef.current = null; + + try { + let audioBuffer: AudioBuffer | undefined = KGAudioInterface.instance().getAudioBuffer( + trackId, + audioRegion.getAudioFileId() + ); + + if (!audioBuffer) { + const arrayBuffer = await KGAudioFileStorage.loadAudioFile( + projectName, + audioRegion.getAudioFileId() + ); + if (cancelled) return; + const actx = Tone.getContext().rawContext as AudioContext; + audioBuffer = await actx.decodeAudioData(arrayBuffer); + } + + if (cancelled) return; + + const pcm = audioBuffer.getChannelData(0); + const sampleRate = audioBuffer.sampleRate; + const regionDurationSeconds = (audioRegion.getLength() * 60) / bpm; + + const worker = new Worker( + new URL('../../workers/spectrogramWorker.ts', import.meta.url), + { type: 'module' } + ); + workerRef.current = worker; + + worker.onmessage = (e: MessageEvent) => { + if (cancelled) { worker.terminate(); return; } + rawResultRef.current = e.data; + sampleRateRef.current = sampleRate; + regionDurationRef.current = regionDurationSeconds; + renderSpectrogram(e.data, sampleRate, regionDurationSeconds, thresholdDb, power); + setLoading(false); + worker.terminate(); + workerRef.current = null; + }; + + const request: SpectrogramRequest = { + pcm: pcm.slice(0) as Float32Array, + sampleRate, + clipStartOffsetSeconds: audioRegion.getClipStartOffsetSeconds(), + regionDurationSeconds, + bpm, + }; + worker.postMessage(request, [request.pcm.buffer]); + } catch (err) { + if (!cancelled) { + console.error('SpectrogramCanvas: failed to compute spectrogram', err); + setLoading(false); + } + } + }; + + compute(); + + return () => { + cancelled = true; + workerRef.current?.terminate(); + workerRef.current = null; + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [audioRegion, trackId, projectName, bpm]); + + return ( + <> + + {loading && ( +
+ Computing spectrogram… +
+ )} + + ); +}; + +export default SpectrogramCanvas; diff --git a/src/components/track/Region.css b/src/components/track/Region.css index 9175075..ff97412 100644 --- a/src/components/track/Region.css +++ b/src/components/track/Region.css @@ -119,6 +119,27 @@ background: rgba(0, 0, 0, 0.35); } +.region-spectrogram-btn { + position: absolute; + top: 4px; + left: 4px; + background: rgba(0, 0, 0, 0.25); + color: #fff; + border: 1px solid rgba(255, 255, 255, 0.2); + border-radius: 3px; + padding: 2px; + margin: 0; + cursor: pointer; + z-index: 2; + display: inline-flex; + align-items: center; + justify-content: center; +} + +.region-spectrogram-btn:hover { + background: rgba(0, 0, 0, 0.35); +} + /* Instrument dropdown specific styles */ .instrument-dropdown .quant-dropdown { min-width: 80px; diff --git a/src/components/track/RegionItem.tsx b/src/components/track/RegionItem.tsx index d6f6934..5a4f0b9 100644 --- a/src/components/track/RegionItem.tsx +++ b/src/components/track/RegionItem.tsx @@ -1,6 +1,7 @@ import React, { useState, useRef, useEffect } from 'react'; import './Region.css'; import { FaPencilAlt } from 'react-icons/fa'; +import { MdGraphicEq } from 'react-icons/md'; import type { ResizeAction } from '../interfaces'; import { REGION_CONSTANTS, DEBUG_MODE } from '../../constants'; import { KGMidiRegion } from '../../core/region/KGMidiRegion'; @@ -27,6 +28,8 @@ interface RegionItemProps { onClick?: (regionId: string) => void; // Explicit open piano roll action from header pencil icon onOpenPianoRoll?: (regionId: string) => void; + // Open spectrogram viewer for audio regions + onOpenSpectrogram?: (regionId: string) => void; // MIDI region data for rendering notes midiRegion?: KGMidiRegion; // Audio region data for rendering waveform @@ -49,6 +52,7 @@ const RegionItem: React.FC = ({ onDragEnd, onClick, onOpenPianoRoll, + onOpenSpectrogram, midiRegion, audioRegion, audioBuffer @@ -576,6 +580,26 @@ const RegionItem: React.FC = ({ )} + {audioRegion && ( + + )} diff --git a/src/components/track/TrackGridItem.tsx b/src/components/track/TrackGridItem.tsx index 08ee19b..8f322d3 100644 --- a/src/components/track/TrackGridItem.tsx +++ b/src/components/track/TrackGridItem.tsx @@ -26,6 +26,7 @@ interface TrackGridItemProps { onRegionDragEnd?: (regionId: string, finalBarNumber: number, finalTrackIndex: number) => void; onRegionClick?: (regionId: string) => void; onOpenPianoRoll?: (regionId: string) => void; + onOpenSpectrogram?: (regionId: string) => void; allTracks?: KGTrack[]; // Added to access all tracks for drag operations onKGOneClipDrop?: (e: React.DragEvent, trackIndex: number) => void; } @@ -47,6 +48,7 @@ const TrackGridItem: React.FC = ({ onRegionDragEnd, onRegionClick, onOpenPianoRoll, + onOpenSpectrogram, allTracks, onKGOneClipDrop, }) => { @@ -566,6 +568,9 @@ const TrackGridItem: React.FC = ({ onRegionClick(regionId); } }} + onOpenSpectrogram={audioRegion ? (regionId) => { + onOpenSpectrogram?.(regionId); + } : undefined} midiRegion={midiRegion} audioRegion={audioRegion} audioBuffer={audioBuffer} diff --git a/src/components/track/TrackGridPanel.tsx b/src/components/track/TrackGridPanel.tsx index da64b29..875895e 100644 --- a/src/components/track/TrackGridPanel.tsx +++ b/src/components/track/TrackGridPanel.tsx @@ -30,6 +30,7 @@ interface TrackGridPanelProps { onRegionUpdated?: (regionId: string, updates: Partial, expectedModelUpdates?: { startBeat: number, length: number }) => void; onRegionClick?: (regionId: string) => void; onOpenPianoRoll?: (regionId: string) => void; + onOpenSpectrogram?: (regionId: string) => void; onExternalDropComplete?: (trackIndex: number, regionUI: RegionUI) => void; } @@ -46,6 +47,7 @@ const TrackGridPanel: React.FC = ({ onRegionUpdated, onRegionClick, onOpenPianoRoll, + onOpenSpectrogram, onExternalDropComplete, }) => { const gridContainerRef = useRef(null); @@ -651,6 +653,7 @@ const TrackGridPanel: React.FC = ({ onRegionDragEnd={handleRegionDragEnd} onRegionClick={handleRegionClick} onOpenPianoRoll={onOpenPianoRoll} + onOpenSpectrogram={onOpenSpectrogram} allTracks={tracks} onKGOneClipDrop={handleExternalDrop} /> diff --git a/src/stores/projectStore.ts b/src/stores/projectStore.ts index 9d1f1a1..0e2f50d 100644 --- a/src/stores/projectStore.ts +++ b/src/stores/projectStore.ts @@ -78,6 +78,7 @@ interface ProjectState { // Piano roll state showPianoRoll: boolean; activeRegionId: string | null; + pianoRollMode: 'midi-edit' | 'spectrogram'; // ChatBox state showChatBox: boolean; @@ -146,6 +147,8 @@ interface ProjectState { // Piano roll actions setShowPianoRoll: (show: boolean) => void; setActiveRegionId: (regionId: string | null) => void; + openMidiPianoRoll: (regionId: string) => void; + openSpectrogramViewer: (regionId: string) => void; // Project state cleanup cleanupProjectState: () => void; @@ -304,6 +307,7 @@ export const useProjectStore = create((set, get) => { // Initial piano roll state showPianoRoll: false, activeRegionId: null, + pianoRollMode: 'midi-edit' as const, // Initial ChatBox state showChatBox: initialChatBoxState, @@ -1028,10 +1032,18 @@ export const useProjectStore = create((set, get) => { setShowPianoRoll: (show: boolean) => { set({ showPianoRoll: show }); }, - + setActiveRegionId: (regionId: string | null) => { set({ activeRegionId: regionId }); }, + + openMidiPianoRoll: (regionId: string) => { + set({ showPianoRoll: true, activeRegionId: regionId, pianoRollMode: 'midi-edit' }); + }, + + openSpectrogramViewer: (regionId: string) => { + set({ showPianoRoll: true, activeRegionId: regionId, pianoRollMode: 'spectrogram' }); + }, // Project state cleanup - used when starting new/loading projects cleanupProjectState: () => { diff --git a/src/workers/spectrogramWorker.ts b/src/workers/spectrogramWorker.ts new file mode 100644 index 0000000..eba58f4 --- /dev/null +++ b/src/workers/spectrogramWorker.ts @@ -0,0 +1,96 @@ +import FFT from 'fft.js'; + +export interface SpectrogramRequest { + pcm: Float32Array; + sampleRate: number; + clipStartOffsetSeconds: number; + regionDurationSeconds: number; + bpm: number; +} + +export interface SpectrogramResult { + data: Float32Array; // [timeSteps × 128] row-major, row = time step, col = pitch 0-127 + timeSteps: number; +} + +const FFT_SIZE = 8192; +const HOP_SIZE = 1024; +const PITCH_BINS = 128; + +function hannWindow(size: number): Float32Array { + const w = new Float32Array(size); + for (let i = 0; i < size; i++) { + w[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / (size - 1))); + } + return w; +} + +self.onmessage = (e: MessageEvent) => { + const { pcm, sampleRate, clipStartOffsetSeconds, regionDurationSeconds } = e.data; + + const startSample = Math.floor(clipStartOffsetSeconds * sampleRate); + const endSample = Math.min(pcm.length, startSample + Math.ceil(regionDurationSeconds * sampleRate)); + const regionSamples = pcm.subarray(startSample, endSample); + + const fft = new FFT(FFT_SIZE); + const hann = hannWindow(FFT_SIZE); + const complexOut = fft.createComplexArray() as number[]; + const inputPadded = new Float32Array(FFT_SIZE); + + const totalHops = Math.max(1, Math.ceil((regionSamples.length - FFT_SIZE) / HOP_SIZE) + 1); + const result = new Float32Array(totalHops * PITCH_BINS); + + let maxVal = 0; + + for (let hop = 0; hop < totalHops; hop++) { + const offset = hop * HOP_SIZE; + + // Fill windowed frame (zero-pad at end if needed) + inputPadded.fill(0); + const available = Math.min(FFT_SIZE, regionSamples.length - offset); + for (let i = 0; i < available; i++) { + inputPadded[i] = regionSamples[offset + i] * hann[i]; + } + + fft.realTransform(complexOut, inputPadded as unknown as number[]); + fft.completeSpectrum(complexOut); + + // Max-pool magnitude into pitch bins: keep the loudest FFT bin per semitone, + // rather than summing. Summing inflates every bin by how many FFT bins land there. + const pitchRow = hop * PITCH_BINS; + const numBins = FFT_SIZE / 2; + + for (let bin = 1; bin < numBins; bin++) { + const freq = (bin * sampleRate) / FFT_SIZE; + if (freq < 20 || freq > 20000) continue; + + // Convert frequency to MIDI pitch; clamp to piano roll range C0–B7 (MIDI 12–107) + const pitch = Math.round(69 + 12 * Math.log2(freq / 440)); + if (pitch < 12 || pitch > 107) continue; + + const re = complexOut[2 * bin]; + const im = complexOut[2 * bin + 1]; + const magnitude = Math.sqrt(re * re + im * im); + + if (magnitude > result[pitchRow + pitch]) { + result[pitchRow + pitch] = magnitude; + } + } + + // Track global max for normalization + for (let p = 0; p < PITCH_BINS; p++) { + if (result[pitchRow + p] > maxVal) maxVal = result[pitchRow + p]; + } + } + + // Linear normalization only — threshold and power curve are applied in the canvas + // renderer so changing them is instant (no need to re-run the FFT). + if (maxVal > 0) { + for (let i = 0; i < result.length; i++) { + result[i] = result[i] / maxVal; + } + } + + const response: SpectrogramResult = { data: result, timeSteps: totalHops }; + self.postMessage(response, [result.buffer]); +}; From ba0e27e50a08ea103eb36a574f421775e9c9c6b1 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Fri, 1 May 2026 17:55:14 -0700 Subject: [PATCH 02/14] fix: few misc issues when switching between MIDI region and audio regions --- src/components/MainContent.tsx | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/components/MainContent.tsx b/src/components/MainContent.tsx index 444155a..ef27e98 100644 --- a/src/components/MainContent.tsx +++ b/src/components/MainContent.tsx @@ -42,6 +42,7 @@ const MainContent: React.FC = ({ setShowPianoRoll, setActiveRegionId, pianoRollMode, + openMidiPianoRoll, openSpectrogramViewer, addTrack, addAudioTrack, @@ -511,6 +512,16 @@ const MainContent: React.FC = ({ const track = tracks.find(t => t.getId().toString() === region.trackId); if (!track) return; setSelectedTrack(track.getId().toString()); + + // If the piano roll window is already open, follow the selected region's type + if (showPianoRoll) { + const coreRegion = track.getRegions().find(r => r.getId() === regionId); + if (coreRegion?.getCurrentType() === 'KGAudioRegion') { + openSpectrogramViewer(regionId); + } else if (coreRegion?.getCurrentType() === 'KGMidiRegion') { + openMidiPianoRoll(regionId); + } + } }; // Handle explicit pencil action: select region and open piano roll @@ -531,9 +542,8 @@ const MainContent: React.FC = ({ // Reuse selection logic handleRegionClick(regionId); - // Activate and show piano roll - setActiveRegionId(regionId); - setShowPianoRoll(true); + // Activate and show piano roll in midi-edit mode + openMidiPianoRoll(regionId); }; // Handle spectrogram viewer open From 83034455ccb3b421750a3cac61b7118d53b15909 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Fri, 1 May 2026 18:28:45 -0700 Subject: [PATCH 03/14] feat: implemented piano roll hybrid mode to allow user edit MIDI notes while observing spectrogram as reference --- src/components/MainContent.tsx | 65 +++++++++++++------ src/components/piano-roll/PianoGrid.tsx | 1 + src/components/piano-roll/PianoRoll.css | 12 +++- src/components/piano-roll/PianoRoll.tsx | 8 ++- .../piano-roll/PianoRollContent.tsx | 2 +- .../piano-roll/PianoRollToolbar.tsx | 43 ++++++------ src/components/track/Region.css | 21 ++++++ src/components/track/RegionItem.tsx | 29 ++++++++- src/components/track/TrackGridItem.tsx | 8 +++ src/components/track/TrackGridPanel.tsx | 9 +++ src/stores/projectStore.ts | 19 ++++-- 11 files changed, 162 insertions(+), 55 deletions(-) diff --git a/src/components/MainContent.tsx b/src/components/MainContent.tsx index ef27e98..1003b19 100644 --- a/src/components/MainContent.tsx +++ b/src/components/MainContent.tsx @@ -44,6 +44,8 @@ const MainContent: React.FC = ({ pianoRollMode, openMidiPianoRoll, openSpectrogramViewer, + openHybridMode, + hybridAudioRegionId, addTrack, addAudioTrack, projectName, @@ -552,6 +554,19 @@ const MainContent: React.FC = ({ openSpectrogramViewer(regionId); }; + // Handle hybrid mode open (+ button clicked on opposite-type region) + const handleOpenHybrid = (regionId: string) => { + if (pianoRollMode === 'midi-edit' && activeRegionId) { + openHybridMode(activeRegionId, regionId); + } else if (pianoRollMode === 'spectrogram' && activeRegionId) { + openHybridMode(regionId, activeRegionId); + } + }; + + // + button is visible only when piano roll is open and mode is not hybrid + const showHybridButtonForAudio = showPianoRoll && pianoRollMode === 'midi-edit'; + const showHybridButtonForMidi = showPianoRoll && pianoRollMode === 'spectrogram'; + // Handle piano roll close const handlePianoRollClose = () => { setShowPianoRoll(false); @@ -844,6 +859,9 @@ const MainContent: React.FC = ({ onRegionClick={handleRegionClick} onOpenPianoRoll={handleOpenPianoRoll} onOpenSpectrogram={handleOpenSpectrogram} + showHybridButtonForAudio={showHybridButtonForAudio} + showHybridButtonForMidi={showHybridButtonForMidi} + onOpenHybrid={handleOpenHybrid} onExternalDropComplete={handleExternalDropComplete} /> @@ -855,26 +873,33 @@ const MainContent: React.FC = ({ onClose={handlePianoRollClose} regionId={activeRegionId} mode={pianoRollMode} - audioRegion={pianoRollMode === 'spectrogram' && activeRegionId - ? (() => { - for (const track of tracks) { - const region = track.getRegions().find(r => r.getId() === activeRegionId); - if (region && region.getCurrentType() === 'KGAudioRegion') { - return region as unknown as KGAudioRegion; - } - } - return undefined; - })() - : undefined} - trackId={pianoRollMode === 'spectrogram' && activeRegionId - ? (() => { - for (const track of tracks) { - const region = track.getRegions().find(r => r.getId() === activeRegionId); - if (region) return track.getId().toString(); - } - return undefined; - })() - : undefined} + audioRegion={(() => { + // spectrogram mode: audio region IS the activeRegionId + // hybrid mode: audio region is hybridAudioRegionId + const audioId = pianoRollMode === 'spectrogram' ? activeRegionId + : pianoRollMode === 'hybrid' ? hybridAudioRegionId + : null; + if (!audioId) return undefined; + for (const track of tracks) { + const region = track.getRegions().find(r => r.getId() === audioId); + if (region && region.getCurrentType() === 'KGAudioRegion') { + return region as unknown as KGAudioRegion; + } + } + return undefined; + })()} + trackId={(() => { + const audioId = pianoRollMode === 'spectrogram' ? activeRegionId + : pianoRollMode === 'hybrid' ? hybridAudioRegionId + : null; + if (!audioId) return undefined; + for (const track of tracks) { + if (track.getRegions().some(r => r.getId() === audioId)) { + return track.getId().toString(); + } + } + return undefined; + })()} projectName={savedProjectName} />, document.body diff --git a/src/components/piano-roll/PianoGrid.tsx b/src/components/piano-roll/PianoGrid.tsx index b21d2c3..e49d197 100644 --- a/src/components/piano-roll/PianoGrid.tsx +++ b/src/components/piano-roll/PianoGrid.tsx @@ -32,6 +32,7 @@ interface PianoGridProps { bpm?: number; spectrogramThresholdDb?: number; spectrogramPower?: number; + mode?: 'midi-edit' | 'spectrogram' | 'hybrid'; } interface CursorPosition { diff --git a/src/components/piano-roll/PianoRoll.css b/src/components/piano-roll/PianoRoll.css index d4929b7..58c8fdb 100644 --- a/src/components/piano-roll/PianoRoll.css +++ b/src/components/piano-roll/PianoRoll.css @@ -77,10 +77,17 @@ /* Ensure toolbar and its dropdowns appear above piano roll content */ } +.piano-roll-toolbar .tool-button { + width: 20px; + height: 20px; +} + /* Override pointer-events for piano roll toolbar sections */ .piano-roll-toolbar .toolbar-left, .piano-roll-toolbar .toolbar-right { pointer-events: auto; + /* max-width: 70%; */ + width: auto; } .piano-roll-toolbar .quant-button { @@ -101,6 +108,7 @@ display: flex; align-items: center; gap: 6px; + margin-left: 5px; } .spectrogram-control-label { @@ -121,7 +129,7 @@ .spectrogram-threshold-value { font-size: 10px; color: #e0e0e0; - min-width: 44px; + min-width: 30px; text-align: right; white-space: nowrap; } @@ -131,7 +139,7 @@ text-align: center; font-size: 14px; color: #e0e0e0; - text-transform: uppercase; + /* text-transform: uppercase; */ cursor: pointer; padding: 5px; border-radius: 3px; diff --git a/src/components/piano-roll/PianoRoll.tsx b/src/components/piano-roll/PianoRoll.tsx index 990aed1..a63af15 100644 --- a/src/components/piano-roll/PianoRoll.tsx +++ b/src/components/piano-roll/PianoRoll.tsx @@ -23,7 +23,7 @@ interface PianoRollProps { regionId: string | null; initialPosition?: { x: number; y: number }; initialSize?: { width: number; height: number }; - mode?: 'midi-edit' | 'spectrogram'; + mode?: 'midi-edit' | 'spectrogram' | 'hybrid'; audioRegion?: KGAudioRegion; trackId?: string; projectName?: string; @@ -40,6 +40,7 @@ const PianoRoll: React.FC = ({ projectName, }) => { const isSpectrogram = mode === 'spectrogram'; + const isHybrid = mode === 'hybrid'; const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled, bpm } = useProjectStore(); // Tool state for piano roll @@ -870,6 +871,11 @@ const PianoRoll: React.FC = ({ // Get the title for the piano roll based on the active region const getPianoRollTitle = () => { if (isSpectrogram) return audioRegion ? `SPECTROGRAM — ${audioRegion.getName()}` : 'SPECTROGRAM'; + if (isHybrid) { + const midiName = activeRegion?.getName() ?? 'MIDI'; + const audioName = audioRegion?.getName() ?? 'Audio'; + return `${midiName} + ${audioName}`; + } if (!activeRegion) return "EDIT NOTE CLIP"; // Calculate the bar and beat position of the region diff --git a/src/components/piano-roll/PianoRollContent.tsx b/src/components/piano-roll/PianoRollContent.tsx index 5f47117..24f7c56 100644 --- a/src/components/piano-roll/PianoRollContent.tsx +++ b/src/components/piano-roll/PianoRollContent.tsx @@ -27,7 +27,7 @@ interface PianoRollContentProps { selectedMode: string; keySignature: KeySignature; chordGuide: string; - mode?: 'midi-edit' | 'spectrogram'; + mode?: 'midi-edit' | 'spectrogram' | 'hybrid'; audioRegion?: KGAudioRegion; trackId?: string; projectName?: string; diff --git a/src/components/piano-roll/PianoRollToolbar.tsx b/src/components/piano-roll/PianoRollToolbar.tsx index 4df21b6..d0746e1 100644 --- a/src/components/piano-roll/PianoRollToolbar.tsx +++ b/src/components/piano-roll/PianoRollToolbar.tsx @@ -24,7 +24,7 @@ interface PianoRollToolbarProps { chordGuide: string; onChordGuideChange: (value: string) => void; blinkButton?: string | null; - mode?: 'midi-edit' | 'spectrogram'; + mode?: 'midi-edit' | 'spectrogram' | 'hybrid'; thresholdDb?: number; onThresholdChange?: (db: number) => void; power?: number; @@ -51,11 +51,27 @@ const PianoRollToolbar: React.FC = ({ onPowerChange, }) => { const isSpectrogram = mode === 'spectrogram'; + const showMidiControls = mode !== 'spectrogram'; // midi-edit and hybrid + const showSpecControls = mode === 'spectrogram' || mode === 'hybrid'; return (
- {!isSpectrogram && ( + {showMidiControls && (
+ + ({ label: data.name, value: id }))} value={selectedMode} @@ -80,27 +96,8 @@ const PianoRollToolbar: React.FC = ({
)} - {!isSpectrogram && ( -
- - -
- )} -
- {!isSpectrogram && ( + {showMidiControls && ( <> = ({ )} - {isSpectrogram && ( + {showSpecControls && (
Floor void; // Open spectrogram viewer for audio regions onOpenSpectrogram?: (regionId: string) => void; + // Enter hybrid mode (show + when piano roll is open with the opposite region type selected) + showHybridButton?: boolean; + onOpenHybrid?: (regionId: string) => void; // MIDI region data for rendering notes midiRegion?: KGMidiRegion; // Audio region data for rendering waveform @@ -53,6 +56,8 @@ const RegionItem: React.FC = ({ onClick, onOpenPianoRoll, onOpenSpectrogram, + showHybridButton, + onOpenHybrid, midiRegion, audioRegion, audioBuffer @@ -64,7 +69,7 @@ const RegionItem: React.FC = ({ const [resizeEdge, setResizeEdge] = useState('none'); const [isResizing, setIsResizing] = useState(false); const [isDragging, setIsDragging] = useState(false); - const initialMousePosRef = useRef<{x: number, y: number}>({x: 0, y: 0}); + const initialMousePosRef = useRef<{ x: number, y: number }>({ x: 0, y: 0 }); // Use refs to track states for immediate access const isResizingRef = useRef(false); const isDraggingRef = useRef(false); @@ -600,6 +605,26 @@ const RegionItem: React.FC = ({ )} + {showHybridButton && ( + + )}
diff --git a/src/components/track/TrackGridItem.tsx b/src/components/track/TrackGridItem.tsx index 8f322d3..20e891c 100644 --- a/src/components/track/TrackGridItem.tsx +++ b/src/components/track/TrackGridItem.tsx @@ -27,6 +27,9 @@ interface TrackGridItemProps { onRegionClick?: (regionId: string) => void; onOpenPianoRoll?: (regionId: string) => void; onOpenSpectrogram?: (regionId: string) => void; + showHybridButtonForAudio?: boolean; + showHybridButtonForMidi?: boolean; + onOpenHybrid?: (regionId: string) => void; allTracks?: KGTrack[]; // Added to access all tracks for drag operations onKGOneClipDrop?: (e: React.DragEvent, trackIndex: number) => void; } @@ -49,6 +52,9 @@ const TrackGridItem: React.FC = ({ onRegionClick, onOpenPianoRoll, onOpenSpectrogram, + showHybridButtonForAudio, + showHybridButtonForMidi, + onOpenHybrid, allTracks, onKGOneClipDrop, }) => { @@ -571,6 +577,8 @@ const TrackGridItem: React.FC = ({ onOpenSpectrogram={audioRegion ? (regionId) => { onOpenSpectrogram?.(regionId); } : undefined} + showHybridButton={audioRegion ? showHybridButtonForAudio : showHybridButtonForMidi} + onOpenHybrid={onOpenHybrid} midiRegion={midiRegion} audioRegion={audioRegion} audioBuffer={audioBuffer} diff --git a/src/components/track/TrackGridPanel.tsx b/src/components/track/TrackGridPanel.tsx index 875895e..43718bc 100644 --- a/src/components/track/TrackGridPanel.tsx +++ b/src/components/track/TrackGridPanel.tsx @@ -31,6 +31,9 @@ interface TrackGridPanelProps { onRegionClick?: (regionId: string) => void; onOpenPianoRoll?: (regionId: string) => void; onOpenSpectrogram?: (regionId: string) => void; + showHybridButtonForAudio?: boolean; + showHybridButtonForMidi?: boolean; + onOpenHybrid?: (regionId: string) => void; onExternalDropComplete?: (trackIndex: number, regionUI: RegionUI) => void; } @@ -48,6 +51,9 @@ const TrackGridPanel: React.FC = ({ onRegionClick, onOpenPianoRoll, onOpenSpectrogram, + showHybridButtonForAudio, + showHybridButtonForMidi, + onOpenHybrid, onExternalDropComplete, }) => { const gridContainerRef = useRef(null); @@ -654,6 +660,9 @@ const TrackGridPanel: React.FC = ({ onRegionClick={handleRegionClick} onOpenPianoRoll={onOpenPianoRoll} onOpenSpectrogram={onOpenSpectrogram} + showHybridButtonForAudio={showHybridButtonForAudio} + showHybridButtonForMidi={showHybridButtonForMidi} + onOpenHybrid={onOpenHybrid} allTracks={tracks} onKGOneClipDrop={handleExternalDrop} /> diff --git a/src/stores/projectStore.ts b/src/stores/projectStore.ts index 0e2f50d..836f557 100644 --- a/src/stores/projectStore.ts +++ b/src/stores/projectStore.ts @@ -78,7 +78,8 @@ interface ProjectState { // Piano roll state showPianoRoll: boolean; activeRegionId: string | null; - pianoRollMode: 'midi-edit' | 'spectrogram'; + pianoRollMode: 'midi-edit' | 'spectrogram' | 'hybrid'; + hybridAudioRegionId: string | null; // ChatBox state showChatBox: boolean; @@ -149,6 +150,7 @@ interface ProjectState { setActiveRegionId: (regionId: string | null) => void; openMidiPianoRoll: (regionId: string) => void; openSpectrogramViewer: (regionId: string) => void; + openHybridMode: (midiRegionId: string, audioRegionId: string) => void; // Project state cleanup cleanupProjectState: () => void; @@ -308,6 +310,7 @@ export const useProjectStore = create((set, get) => { showPianoRoll: false, activeRegionId: null, pianoRollMode: 'midi-edit' as const, + hybridAudioRegionId: null, // Initial ChatBox state showChatBox: initialChatBoxState, @@ -1038,20 +1041,24 @@ export const useProjectStore = create((set, get) => { }, openMidiPianoRoll: (regionId: string) => { - set({ showPianoRoll: true, activeRegionId: regionId, pianoRollMode: 'midi-edit' }); + set({ showPianoRoll: true, activeRegionId: regionId, pianoRollMode: 'midi-edit', hybridAudioRegionId: null }); }, openSpectrogramViewer: (regionId: string) => { - set({ showPianoRoll: true, activeRegionId: regionId, pianoRollMode: 'spectrogram' }); + set({ showPianoRoll: true, activeRegionId: regionId, pianoRollMode: 'spectrogram', hybridAudioRegionId: null }); + }, + + openHybridMode: (midiRegionId: string, audioRegionId: string) => { + set({ showPianoRoll: true, activeRegionId: midiRegionId, hybridAudioRegionId: audioRegionId, pianoRollMode: 'hybrid' }); }, // Project state cleanup - used when starting new/loading projects cleanupProjectState: () => { // Close piano roll if it's visible set({ showPianoRoll: false }); - - // Clear active region - set({ activeRegionId: null }); + + // Clear active region and hybrid state + set({ activeRegionId: null, hybridAudioRegionId: null, pianoRollMode: 'midi-edit' }); // Clear any selected items KGCore.instance().clearSelectedItems(); From b7926300e18cc5ed9f43491c3771b7caeebfbaf1 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Fri, 1 May 2026 18:33:25 -0700 Subject: [PATCH 04/14] fix: unit test errors and TS type errors --- src/core/project-upgrader/upgradeToV7.test.ts | 4 ++-- src/core/track/KGMidiTrack.test.ts | 4 ++-- src/workers/spectrogramWorker.ts | 11 +++++++++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/core/project-upgrader/upgradeToV7.test.ts b/src/core/project-upgrader/upgradeToV7.test.ts index 6df72bd..3bc2b19 100644 --- a/src/core/project-upgrader/upgradeToV7.test.ts +++ b/src/core/project-upgrader/upgradeToV7.test.ts @@ -41,10 +41,10 @@ describe('upgradeToV7', () => { }); it('clamps values that exceed MAX_TRACK_VOLUME_DB', () => { - // linear > 1.0 would give positive dB; cap at +6 + // linear > 1.0 gives positive dB but 2.0 is still below the +12 dB ceiling const project = makeProject([2.0]); upgradeToV7(project); - expect(project.getTracks()[0].getVolume()).toBe(AUDIO_INTERFACE_CONSTANTS.MAX_TRACK_VOLUME_DB); + expect(project.getTracks()[0].getVolume()).toBeCloseTo(20 * Math.log10(2.0), 5); }); it('bumps project structure version to 7', () => { diff --git a/src/core/track/KGMidiTrack.test.ts b/src/core/track/KGMidiTrack.test.ts index 73092a3..6d5aaec 100644 --- a/src/core/track/KGMidiTrack.test.ts +++ b/src/core/track/KGMidiTrack.test.ts @@ -348,7 +348,7 @@ describe('KGMidiTrack', () => { // Volume outside valid dB range is clamped track.setVolume(10); - expect(track.getVolume()).toBe(6); + expect(track.getVolume()).toBe(10); track.setVolume(-100); expect(track.getVolume()).toBe(-60); @@ -401,4 +401,4 @@ describe('KGMidiTrack', () => { }); }); }); -}); \ No newline at end of file +}); diff --git a/src/workers/spectrogramWorker.ts b/src/workers/spectrogramWorker.ts index eba58f4..a3a914f 100644 --- a/src/workers/spectrogramWorker.ts +++ b/src/workers/spectrogramWorker.ts @@ -1,5 +1,12 @@ import FFT from 'fft.js'; +type WorkerScopeLike = typeof globalThis & { + onmessage: ((event: MessageEvent) => void) | null; + postMessage: (message: SpectrogramResult, transfer: Transferable[]) => void; +}; + +const workerScope = self as WorkerScopeLike; + export interface SpectrogramRequest { pcm: Float32Array; sampleRate: number; @@ -25,7 +32,7 @@ function hannWindow(size: number): Float32Array { return w; } -self.onmessage = (e: MessageEvent) => { +workerScope.onmessage = (e: MessageEvent) => { const { pcm, sampleRate, clipStartOffsetSeconds, regionDurationSeconds } = e.data; const startSample = Math.floor(clipStartOffsetSeconds * sampleRate); @@ -92,5 +99,5 @@ self.onmessage = (e: MessageEvent) => { } const response: SpectrogramResult = { data: result, timeSteps: totalHops }; - self.postMessage(response, [result.buffer]); + workerScope.postMessage(response, [result.buffer]); }; From 756fc1bc6ca15da0f23e9b5c15408197eb6a97e4 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Fri, 1 May 2026 18:49:28 -0700 Subject: [PATCH 05/14] fix: minor UI updates --- src/components/piano-roll/PianoRoll.css | 3 ++- src/components/piano-roll/PianoRollHeader.tsx | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/components/piano-roll/PianoRoll.css b/src/components/piano-roll/PianoRoll.css index 58c8fdb..6c421d1 100644 --- a/src/components/piano-roll/PianoRoll.css +++ b/src/components/piano-roll/PianoRoll.css @@ -137,7 +137,8 @@ .piano-roll-title { flex: 1; text-align: center; - font-size: 14px; + font-size: 12px; + font-weight: bold; color: #e0e0e0; /* text-transform: uppercase; */ cursor: pointer; diff --git a/src/components/piano-roll/PianoRollHeader.tsx b/src/components/piano-roll/PianoRollHeader.tsx index ba21bc4..53f05b9 100644 --- a/src/components/piano-roll/PianoRollHeader.tsx +++ b/src/components/piano-roll/PianoRollHeader.tsx @@ -19,12 +19,6 @@ const PianoRollHeader: React.FC = ({ className="piano-roll-header" onMouseDown={onMouseDown} > -
= ({ > {title}
+
); }; From cbd2fc60a09cf2e58ba2bb0604aa1e2c998526e6 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Fri, 1 May 2026 20:18:32 -0700 Subject: [PATCH 06/14] fix: updated the spectrogram icon --- src/components/track/Region.css | 21 +++++++++++++++++++++ src/components/track/RegionItem.tsx | 23 ++++++++++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/components/track/Region.css b/src/components/track/Region.css index b24ef17..642bb4e 100644 --- a/src/components/track/Region.css +++ b/src/components/track/Region.css @@ -119,6 +119,27 @@ background: rgba(0, 0, 0, 0.35); } +.region-waveform-btn { + position: absolute; + top: 4px; + left: 4px; + background: rgba(0, 0, 0, 0.25); + color: #fff; + border: 1px solid rgba(255, 255, 255, 0.2); + border-radius: 3px; + padding: 2px; + margin: 0; + cursor: pointer; + z-index: 2; + display: none; + align-items: center; + justify-content: center; +} + +.region-waveform-btn:hover { + background: rgba(0, 0, 0, 0.35); +} + .region-spectrogram-btn { position: absolute; top: 4px; diff --git a/src/components/track/RegionItem.tsx b/src/components/track/RegionItem.tsx index fdcfca8..1a7b1ee 100644 --- a/src/components/track/RegionItem.tsx +++ b/src/components/track/RegionItem.tsx @@ -585,6 +585,23 @@ const RegionItem: React.FC = ({ )} + {audioRegion && ( + + )} {audioRegion && ( )} {showHybridButton && ( From d5795c528142b45e28968c6239931dec3567282e Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Fri, 1 May 2026 20:34:21 -0700 Subject: [PATCH 07/14] fix: random not able to exit piano roll hybrid mode issue when switching between different MIDI regions (introduced a region dragging start threshold) --- src/components/MainContent.tsx | 7 +- src/components/track/RegionItem.test.tsx | 97 ++++++++++++++++++++++++ src/components/track/RegionItem.tsx | 70 +++++++++-------- src/components/track/TrackGridItem.tsx | 9 ++- src/components/track/TrackGridPanel.tsx | 9 ++- src/stores/projectStore.test.ts | 78 +++++++++++++++++++ 6 files changed, 234 insertions(+), 36 deletions(-) create mode 100644 src/components/track/RegionItem.test.tsx create mode 100644 src/stores/projectStore.test.ts diff --git a/src/components/MainContent.tsx b/src/components/MainContent.tsx index 1003b19..0d9e6eb 100644 --- a/src/components/MainContent.tsx +++ b/src/components/MainContent.tsx @@ -439,8 +439,9 @@ const MainContent: React.FC = ({ } } - // If piano roll is visible, set this region as the active region - if (showPianoRoll) { + // Keep the active piano roll region in sync when that same region is updated. + // Do not switch the editor to an unrelated region from generic move/resize updates. + if (showPianoRoll && activeRegionId === regionId) { setActiveRegionId(regionId); if (DEBUG_MODE.MAIN_CONTENT) { @@ -908,4 +909,4 @@ const MainContent: React.FC = ({ ); }; -export default MainContent; \ No newline at end of file +export default MainContent; diff --git a/src/components/track/RegionItem.test.tsx b/src/components/track/RegionItem.test.tsx new file mode 100644 index 0000000..7172df7 --- /dev/null +++ b/src/components/track/RegionItem.test.tsx @@ -0,0 +1,97 @@ +import React from 'react'; +import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest'; +import { render, fireEvent } from '@testing-library/react'; +import RegionItem from './RegionItem'; +import { KGMidiRegion } from '../../core/region/KGMidiRegion'; +import { KGMainContentState } from '../../core/state/KGMainContentState'; + +vi.mock('../../stores/projectStore', () => ({ + useProjectStore: () => ({ + selectedRegionIds: [], + timeSignature: { numerator: 4, denominator: 4 }, + bpm: 120, + }), +})); + +describe('RegionItem', () => { + beforeAll(() => { + Object.defineProperty(HTMLCanvasElement.prototype, 'getContext', { + value: vi.fn(() => ({ + clearRect: vi.fn(), + fillRect: vi.fn(), + })), + }); + + class ResizeObserverMock { + observe() {} + unobserve() {} + disconnect() {} + } + + vi.stubGlobal('ResizeObserver', ResizeObserverMock); + }); + + beforeEach(() => { + KGMainContentState.instance().setActiveTool('pointer'); + }); + + const renderRegion = (props: Partial> = {}) => { + const midiRegion = new KGMidiRegion('midi-1', 'track-1', 0, 'Test Region', 0, 4); + + return render( + + ); + }; + + it('treats small pointer jitter as a click', () => { + const onClick = vi.fn(); + const onDragStart = vi.fn(); + const onDrag = vi.fn(); + const onDragEnd = vi.fn(); + + const { container } = renderRegion({ onClick, onDragStart, onDrag, onDragEnd }); + const region = container.querySelector('.track-region'); + + expect(region).toBeTruthy(); + + fireEvent.mouseDown(region!, { clientX: 100, clientY: 100 }); + fireEvent.mouseMove(document, { clientX: 102, clientY: 102 }); + fireEvent.mouseUp(document, { clientX: 102, clientY: 102 }); + + expect(onClick).toHaveBeenCalledWith('midi-1'); + expect(onDragStart).not.toHaveBeenCalled(); + expect(onDrag).not.toHaveBeenCalled(); + expect(onDragEnd).not.toHaveBeenCalled(); + }); + + it('starts a drag after crossing the movement threshold', () => { + const onClick = vi.fn(); + const onDragStart = vi.fn(); + const onDrag = vi.fn(); + const onDragEnd = vi.fn(); + + const { container } = renderRegion({ onClick, onDragStart, onDrag, onDragEnd }); + const region = container.querySelector('.track-region'); + + expect(region).toBeTruthy(); + + fireEvent.mouseDown(region!, { clientX: 100, clientY: 100 }); + fireEvent.mouseMove(document, { clientX: 110, clientY: 100 }); + fireEvent.mouseUp(document, { clientX: 110, clientY: 100 }); + + expect(onClick).not.toHaveBeenCalled(); + expect(onDragStart).toHaveBeenCalledWith('midi-1', 100, 100); + expect(onDrag).toHaveBeenCalledWith('midi-1', 10, 0); + expect(onDragEnd).toHaveBeenCalledWith('midi-1'); + }); +}); diff --git a/src/components/track/RegionItem.tsx b/src/components/track/RegionItem.tsx index 1a7b1ee..da5398c 100644 --- a/src/components/track/RegionItem.tsx +++ b/src/components/track/RegionItem.tsx @@ -9,6 +9,8 @@ import { KGAudioRegion } from '../../core/region/KGAudioRegion'; import { useProjectStore } from '../../stores/projectStore'; import { KGMainContentState } from '../../core/state/KGMainContentState'; +const DRAG_START_THRESHOLD_PX = 4; + interface RegionItemProps { id: string; name: string; @@ -73,7 +75,7 @@ const RegionItem: React.FC = ({ // Use refs to track states for immediate access const isResizingRef = useRef(false); const isDraggingRef = useRef(false); - const hasMovedRef = useRef(false); + const isPendingDragRef = useRef(false); // Canvas ref for note visualization const canvasRef = useRef(null); @@ -391,7 +393,7 @@ const RegionItem: React.FC = ({ const activeTool = KGMainContentState.instance().getActiveTool(); if (activeTool === 'pencil') { // Still allow click events to pass through for region selection - if (!hasMovedRef.current && onClick) { + if (onClick) { if (DEBUG_MODE.REGION_ITEM) { console.log(`REGION CLICKED (pencil mode): regionId=${id}`); } @@ -404,7 +406,7 @@ const RegionItem: React.FC = ({ e.preventDefault(); // Reset movement tracking - hasMovedRef.current = false; + isPendingDragRef.current = false; // Store initial mouse position initialMousePosRef.current = { x: e.clientX, y: e.clientY }; @@ -423,21 +425,12 @@ const RegionItem: React.FC = ({ onResizeStart(id, resizeEdge, e.clientX); } } else { - // Start dragging + // Wait for actual pointer movement before promoting this gesture to a drag. if (DEBUG_MODE.REGION_ITEM) { - console.log(`DRAG START: regionId=${id}`); + console.log(`PENDING REGION INTERACTION: regionId=${id}`); } - setIsDragging(true); - isDraggingRef.current = true; - - // Change cursor to grabbing during drag - setCursor('grabbing'); - - // Call the onDragStart callback if provided - if (onDragStart) { - onDragStart(id, e.clientX, e.clientY); - } + isPendingDragRef.current = true; } // Add global event listeners for mouse move and up @@ -447,9 +440,6 @@ const RegionItem: React.FC = ({ // Handle global mouse move for resize or drag const handleGlobalMouseMove = (e: MouseEvent) => { - // Set the hasMovedRef to true as soon as there's movement - hasMovedRef.current = true; - if (isResizingRef.current) { // Handle resize if (DEBUG_MODE.REGION_ITEM) { @@ -463,16 +453,35 @@ const RegionItem: React.FC = ({ if (onResize) { onResize(id, resizeEdge, deltaX); } - } else if (isDraggingRef.current) { + } else if (isDraggingRef.current || isPendingDragRef.current) { + const deltaX = e.clientX - initialMousePosRef.current.x; + const deltaY = e.clientY - initialMousePosRef.current.y; + const movedEnough = Math.hypot(deltaX, deltaY) >= DRAG_START_THRESHOLD_PX; + + if (!isDraggingRef.current) { + if (!movedEnough) { + return; + } + + if (DEBUG_MODE.REGION_ITEM) { + console.log(`DRAG START: regionId=${id}`); + } + + isPendingDragRef.current = false; + setIsDragging(true); + isDraggingRef.current = true; + setCursor('grabbing'); + + if (onDragStart) { + onDragStart(id, initialMousePosRef.current.x, initialMousePosRef.current.y); + } + } + // Handle drag if (DEBUG_MODE.REGION_ITEM) { console.log(`DRAG MOVE: regionId=${id}, trackIndex=${trackIndex}`); } - // Calculate delta from initial position - const deltaX = e.clientX - initialMousePosRef.current.x; - const deltaY = e.clientY - initialMousePosRef.current.y; - // Call the onDrag callback if provided if (onDrag) { onDrag(id, deltaX, deltaY); @@ -511,16 +520,15 @@ const RegionItem: React.FC = ({ if (onDragEnd) { onDragEnd(id); } - - // If there was no movement, treat it as a click - if (!hasMovedRef.current && onClick) { - if (DEBUG_MODE.REGION_ITEM) { - console.log(`REGION CLICKED: regionId=${id}`); - } - onClick(id); + } else if (isPendingDragRef.current && onClick) { + if (DEBUG_MODE.REGION_ITEM) { + console.log(`REGION CLICKED: regionId=${id}`); } + onClick(id); } + isPendingDragRef.current = false; + // Remove global event listeners document.removeEventListener('mousemove', handleGlobalMouseMove); document.removeEventListener('mouseup', handleGlobalMouseUp); @@ -652,4 +660,4 @@ const RegionItem: React.FC = ({ ); }; -export default RegionItem; \ No newline at end of file +export default RegionItem; diff --git a/src/components/track/TrackGridItem.tsx b/src/components/track/TrackGridItem.tsx index 20e891c..f85e481 100644 --- a/src/components/track/TrackGridItem.tsx +++ b/src/components/track/TrackGridItem.tsx @@ -492,6 +492,13 @@ const TrackGridItem: React.FC = ({ currentDragTop.current = null; currentDragRegion.current = null; + if (finalBarNumber === region.barNumber && finalTrackIndex === region.trackIndex) { + if (DEBUG_MODE.TRACK_GRID_ITEM) { + console.log(`Skipping no-op drag update for region ${regionId}`); + } + return; + } + // Notify parent about drag end with final values if (onRegionDragEnd) { onRegionDragEnd(regionId, finalBarNumber, finalTrackIndex); @@ -589,4 +596,4 @@ const TrackGridItem: React.FC = ({ ); }; -export default TrackGridItem; \ No newline at end of file +export default TrackGridItem; diff --git a/src/components/track/TrackGridPanel.tsx b/src/components/track/TrackGridPanel.tsx index 43718bc..75be0f6 100644 --- a/src/components/track/TrackGridPanel.tsx +++ b/src/components/track/TrackGridPanel.tsx @@ -386,6 +386,13 @@ const TrackGridPanel: React.FC = ({ // Find the region const region = regions.find(r => r.id === regionId); if (!region) return; + + if (finalBarNumber === region.barNumber && finalTrackIndex === region.trackIndex) { + if (DEBUG_MODE.TRACK_GRID_PANEL) { + console.log(`Skipping no-op move for region ${regionId}`); + } + return; + } // Get the target track const targetTrack = tracks[finalTrackIndex]; @@ -680,4 +687,4 @@ const TrackGridPanel: React.FC = ({ ); }; -export default TrackGridPanel; \ No newline at end of file +export default TrackGridPanel; diff --git a/src/stores/projectStore.test.ts b/src/stores/projectStore.test.ts new file mode 100644 index 0000000..a99935c --- /dev/null +++ b/src/stores/projectStore.test.ts @@ -0,0 +1,78 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { act } from '@testing-library/react'; +import { KGMidiTrack } from '../core/track/KGMidiTrack'; + +const mockProject = { + getTimeSignature: () => ({ numerator: 4, denominator: 4 }), + getMaxBars: () => 32, + getBarWidthMultiplier: () => 1, + getTracks: () => [new KGMidiTrack('Track 1', 0, 'acoustic_grand_piano')], + getBpm: () => 120, + getKeySignature: () => 'C major', + getName: () => 'Test Project', + getSelectedMode: () => 'major', + getIsLooping: () => false, + getLoopingRange: () => null, +}; + +const mockCore = { + getCurrentProject: () => mockProject, + setPlayheadUpdateCallback: vi.fn(), + setPlaybackStateChangeCallback: vi.fn(), + getSelectedItems: () => [], + onSelectionChanged: vi.fn(), + canUndo: () => false, + canRedo: () => false, + getUndoDescription: () => '', + getRedoDescription: () => '', + setOnCommandHistoryChanged: vi.fn(), + executeCommand: vi.fn(), + clearSelectedItems: vi.fn(), + getStatus: () => 'Ready', + getPlayheadPosition: () => 0, + getIsPlaying: () => false, +}; + +vi.mock('../core/KGCore', () => ({ + KGCore: { + instance: () => mockCore, + }, +})); + +vi.mock('../core/config/ConfigManager', () => ({ + ConfigManager: { + instance: () => ({ + getIsInitialized: () => true, + get: () => false, + }), + }, +})); + +describe('projectStore piano roll state', () => { + beforeEach(() => { + vi.resetModules(); + }); + + it('clears hybrid state when opening a MIDI region', async () => { + const { useProjectStore } = await import('./projectStore'); + + act(() => { + useProjectStore.getState().openHybridMode('midi-a', 'audio-a'); + }); + + let state = useProjectStore.getState(); + expect(state.pianoRollMode).toBe('hybrid'); + expect(state.activeRegionId).toBe('midi-a'); + expect(state.hybridAudioRegionId).toBe('audio-a'); + + act(() => { + useProjectStore.getState().openMidiPianoRoll('midi-b'); + }); + + state = useProjectStore.getState(); + expect(state.showPianoRoll).toBe(true); + expect(state.pianoRollMode).toBe('midi-edit'); + expect(state.activeRegionId).toBe('midi-b'); + expect(state.hybridAudioRegionId).toBeNull(); + }); +}); From d59ad68f72fb1d0f1a0201953fe5ec533ae509a1 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Fri, 1 May 2026 23:07:44 -0700 Subject: [PATCH 08/14] feat: added fine tune region position feature --- src/components/track/Region.css | 74 ++++---- src/components/track/RegionItem.tsx | 240 +++++++++++++++--------- src/components/track/TrackGridItem.tsx | 11 ++ src/components/track/TrackGridPanel.tsx | 39 ++++ src/constants/uiConstants.ts | 2 + 5 files changed, 239 insertions(+), 127 deletions(-) diff --git a/src/components/track/Region.css b/src/components/track/Region.css index 642bb4e..ef43a56 100644 --- a/src/components/track/Region.css +++ b/src/components/track/Region.css @@ -97,11 +97,23 @@ background-color: #5a9b6a; } -/* Region pencil trigger inside content */ -.region-pencil-btn { +/* Left-side button cluster inside region-content */ +.region-left-buttons { position: absolute; top: 4px; left: 4px; + display: flex; + flex-direction: row; + align-items: center; + gap: 2px; + z-index: 2; +} + +/* Shared style for all region content buttons */ +.region-pencil-btn, +.region-waveform-btn, +.region-spectrogram-btn, +.region-hybrid-btn { background: rgba(0, 0, 0, 0.25); color: #fff; border: 1px solid rgba(255, 255, 255, 0.2); @@ -109,77 +121,61 @@ padding: 2px; margin: 0; cursor: pointer; - z-index: 2; display: inline-flex; align-items: center; justify-content: center; } -.region-pencil-btn:hover { +.region-pencil-btn:hover, +.region-spectrogram-btn:hover, +.region-hybrid-btn:hover { background: rgba(0, 0, 0, 0.35); } .region-waveform-btn { - position: absolute; - top: 4px; - left: 4px; - background: rgba(0, 0, 0, 0.25); - color: #fff; - border: 1px solid rgba(255, 255, 255, 0.2); - border-radius: 3px; - padding: 2px; - margin: 0; - cursor: pointer; - z-index: 2; display: none; - align-items: center; - justify-content: center; } .region-waveform-btn:hover { background: rgba(0, 0, 0, 0.35); } -.region-spectrogram-btn { - position: absolute; - top: 4px; - left: 4px; +/* Fine-move widget: sits inline in the left button cluster */ +.region-fine-move-widget { + display: flex; + flex-direction: row; + align-items: center; + gap: 2px; +} + +.region-fine-move-btn { background: rgba(0, 0, 0, 0.25); color: #fff; border: 1px solid rgba(255, 255, 255, 0.2); border-radius: 3px; padding: 2px; margin: 0; - cursor: pointer; - z-index: 2; + cursor: ew-resize; display: inline-flex; align-items: center; justify-content: center; + flex-shrink: 0; } -.region-spectrogram-btn:hover { +.region-fine-move-btn:hover { background: rgba(0, 0, 0, 0.35); } -.region-hybrid-btn { - position: absolute; - top: 4px; - left: 24px; +.region-fine-move-label { background: rgba(0, 0, 0, 0.25); color: #fff; border: 1px solid rgba(255, 255, 255, 0.2); border-radius: 3px; - padding: 2px; - margin: 0; - cursor: pointer; - z-index: 2; - display: inline-flex; - align-items: center; - justify-content: center; -} - -.region-hybrid-btn:hover { - background: rgba(0, 0, 0, 0.35); + padding: 2px 4px; + font-size: 10px; + white-space: nowrap; + line-height: 1; + user-select: none; } /* Instrument dropdown specific styles */ diff --git a/src/components/track/RegionItem.tsx b/src/components/track/RegionItem.tsx index da5398c..934a9b2 100644 --- a/src/components/track/RegionItem.tsx +++ b/src/components/track/RegionItem.tsx @@ -1,7 +1,7 @@ import React, { useState, useRef, useEffect } from 'react'; import './Region.css'; import { FaPencilAlt, FaPlus } from 'react-icons/fa'; -import { MdGraphicEq } from 'react-icons/md'; +import { MdGraphicEq, MdSwapHoriz } from 'react-icons/md'; import type { ResizeAction } from '../interfaces'; import { REGION_CONSTANTS, DEBUG_MODE } from '../../constants'; import { KGMidiRegion } from '../../core/region/KGMidiRegion'; @@ -35,6 +35,8 @@ interface RegionItemProps { // Enter hybrid mode (show + when piano roll is open with the opposite region type selected) showHybridButton?: boolean; onOpenHybrid?: (regionId: string) => void; + // Fine-move end callback — passes raw (unscaled) mouse pixel delta + onFineMoveEnd?: (regionId: string, rawPixelDelta: number) => void; // MIDI region data for rendering notes midiRegion?: KGMidiRegion; // Audio region data for rendering waveform @@ -60,6 +62,7 @@ const RegionItem: React.FC = ({ onOpenSpectrogram, showHybridButton, onOpenHybrid, + onFineMoveEnd, midiRegion, audioRegion, audioBuffer @@ -77,6 +80,14 @@ const RegionItem: React.FC = ({ const isDraggingRef = useRef(false); const isPendingDragRef = useRef(false); + // Fine-move state + const [isFineDragging, setIsFineDragging] = useState(false); + const [fineDeltaDisplay, setFineDeltaDisplay] = useState('+0.00'); + const [fineTranslateX, setFineTranslateX] = useState(0); + const isFineDraggingRef = useRef(false); + const fineMouseStartXRef = useRef(0); + const fineRawDeltaRef = useRef(0); + // Canvas ref for note visualization const canvasRef = useRef(null); const regionContentRef = useRef(null); @@ -534,11 +545,48 @@ const RegionItem: React.FC = ({ document.removeEventListener('mouseup', handleGlobalMouseUp); }; + // Fine-move handlers + const handleFineMoveGlobalMouseMove = (e: MouseEvent) => { + if (!isFineDraggingRef.current) return; + const rawDelta = e.clientX - fineMouseStartXRef.current; + fineRawDeltaRef.current = rawDelta; + const scaledDelta = rawDelta * REGION_CONSTANTS.FINE_MOVE_SPEED_RATIO; + setFineTranslateX(scaledDelta); + setFineDeltaDisplay(scaledDelta >= 0 ? `+${scaledDelta.toFixed(2)}` : `${scaledDelta.toFixed(2)}`); + }; + + const handleFineMoveGlobalMouseUp = () => { + if (!isFineDraggingRef.current) return; + isFineDraggingRef.current = false; + setIsFineDragging(false); + setFineTranslateX(0); + document.removeEventListener('mousemove', handleFineMoveGlobalMouseMove); + document.removeEventListener('mouseup', handleFineMoveGlobalMouseUp); + if (fineRawDeltaRef.current !== 0 && onFineMoveEnd) { + onFineMoveEnd(id, fineRawDeltaRef.current); + } + }; + + const handleFineMoveMouseDown = (e: React.MouseEvent) => { + e.preventDefault(); + e.stopPropagation(); + if (isFineDraggingRef.current) return; + fineMouseStartXRef.current = e.clientX; + fineRawDeltaRef.current = 0; + isFineDraggingRef.current = true; + setIsFineDragging(true); + setFineDeltaDisplay('+0.00'); + document.addEventListener('mousemove', handleFineMoveGlobalMouseMove); + document.addEventListener('mouseup', handleFineMoveGlobalMouseUp); + }; + // Clean up event listeners on unmount useEffect(() => { return () => { document.removeEventListener('mousemove', handleGlobalMouseMove); document.removeEventListener('mouseup', handleGlobalMouseUp); + document.removeEventListener('mousemove', handleFineMoveGlobalMouseMove); + document.removeEventListener('mouseup', handleFineMoveGlobalMouseUp); }; }, []); @@ -555,7 +603,7 @@ const RegionItem: React.FC = ({
= ({ {name}
- {!audioRegion && ( - - )} - {audioRegion && ( - - )} - {audioRegion && ( - - )} - {showHybridButton && ( - - )} +
+ {!audioRegion && ( + + )} + {audioRegion && ( + + )} + {audioRegion && ( + + )} + {showHybridButton && ( + + )} +
+ + {isFineDragging && ( + {fineDeltaDisplay} + )} +
+
diff --git a/src/components/track/TrackGridItem.tsx b/src/components/track/TrackGridItem.tsx index f85e481..dbb758c 100644 --- a/src/components/track/TrackGridItem.tsx +++ b/src/components/track/TrackGridItem.tsx @@ -24,6 +24,7 @@ interface TrackGridItemProps { onRegionResizeEnd?: (regionId: string, finalBarNumber: number, finalLength: number) => void; onRegionDrag?: (regionId: string, newBarNumber: number, newTrackIndex: number) => void; onRegionDragEnd?: (regionId: string, finalBarNumber: number, finalTrackIndex: number) => void; + onRegionFineMoveEnd?: (regionId: string, deltaInBars: number) => void; onRegionClick?: (regionId: string) => void; onOpenPianoRoll?: (regionId: string) => void; onOpenSpectrogram?: (regionId: string) => void; @@ -49,6 +50,7 @@ const TrackGridItem: React.FC = ({ onRegionResizeEnd, onRegionDrag, onRegionDragEnd, + onRegionFineMoveEnd, onRegionClick, onOpenPianoRoll, onOpenSpectrogram, @@ -505,6 +507,14 @@ const TrackGridItem: React.FC = ({ } }; + // Handle fine-move end — convert raw pixel delta to delta in bars and pass up + const handleRegionFineMoveEnd = (regionId: string, rawPixelDelta: number) => { + const barWidth = containerWidth / maxBars; + if (barWidth <= 0) return; + const deltaInBars = (rawPixelDelta * REGION_CONSTANTS.FINE_MOVE_SPEED_RATIO) / barWidth; + onRegionFineMoveEnd?.(regionId, deltaInBars); + }; + // Handle region click const handleRegionClick = (regionId: string) => { if (DEBUG_MODE.TRACK_GRID_ITEM) { @@ -570,6 +580,7 @@ const TrackGridItem: React.FC = ({ onDragStart={handleRegionDragStart} onDrag={handleRegionDrag} onDragEnd={handleRegionDragEnd} + onFineMoveEnd={handleRegionFineMoveEnd} // Keep onClick for selection-only logic if needed by parent onClick={handleRegionClick} // New explicit pencil action — disabled for audio regions diff --git a/src/components/track/TrackGridPanel.tsx b/src/components/track/TrackGridPanel.tsx index 75be0f6..c506142 100644 --- a/src/components/track/TrackGridPanel.tsx +++ b/src/components/track/TrackGridPanel.tsx @@ -473,6 +473,44 @@ const TrackGridPanel: React.FC = ({ } }; + // Handle fine-move end — execute MoveRegionCommand with float-precision beat position + const handleRegionFineMoveEnd = (regionId: string, deltaInBars: number) => { + const region = regions.find(r => r.id === regionId); + if (!region) return; + const track = tracks.find(t => t.getId().toString() === region.trackId); + if (!track) return; + const coreRegion = track.getRegions().find(r => r.getId() === regionId); + if (!coreRegion) return; + + const beatsPerBar = timeSignature.numerator; + const newStartFromBeat = Math.max(0, coreRegion.getStartFromBeat() + deltaInBars * beatsPerBar); + if (newStartFromBeat === coreRegion.getStartFromBeat()) return; + + try { + // Use constructor directly (NOT fromBarCoordinates) to preserve float precision + const command = new MoveRegionCommand( + regionId, + newStartFromBeat, + track.getId().toString(), + region.trackIndex + ); + KGCore.instance().executeCommand(command); + + if (DEBUG_MODE.TRACK_GRID_PANEL) { + console.log(`Fine-moved region ${regionId}: startFromBeat=${newStartFromBeat}`); + } + + const newBarNumber = newStartFromBeat / beatsPerBar + 1; + onRegionUpdated?.( + regionId, + { barNumber: newBarNumber, trackId: region.trackId, trackIndex: region.trackIndex }, + { startBeat: newStartFromBeat, length: coreRegion.getLength() } + ); + } catch (error) { + console.error('Error executing fine-move:', error); + } + }; + // Handle region click const handleRegionClick = (regionId: string) => { if (DEBUG_MODE.TRACK_GRID_PANEL) { @@ -664,6 +702,7 @@ const TrackGridPanel: React.FC = ({ onRegionResizeEnd={handleRegionResizeEnd} onRegionDrag={handleRegionDrag} onRegionDragEnd={handleRegionDragEnd} + onRegionFineMoveEnd={handleRegionFineMoveEnd} onRegionClick={handleRegionClick} onOpenPianoRoll={onOpenPianoRoll} onOpenSpectrogram={onOpenSpectrogram} diff --git a/src/constants/uiConstants.ts b/src/constants/uiConstants.ts index a34225f..5bc9bf5 100644 --- a/src/constants/uiConstants.ts +++ b/src/constants/uiConstants.ts @@ -28,6 +28,8 @@ export const REGION_CONSTANTS = { EDGE_THRESHOLD: 10, // Minimum region length in bars MIN_REGION_LENGTH: 1.0, + // Fine-move speed ratio: mouse pixels to region movement pixels + FINE_MOVE_SPEED_RATIO: 0.25, }; // Piano roll related constants From 811b3a0936cfe4eda23bdf4981b8933e43201be4 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Fri, 1 May 2026 23:26:39 -0700 Subject: [PATCH 09/14] feat: added zoom-in/out feature for piano roll window. --- src/components/piano-roll/PianoRoll.css | 28 +++++++++++++ src/components/piano-roll/PianoRoll.tsx | 16 ++++++++ .../piano-roll/PianoRollToolbar.tsx | 41 +++++++++++++++++++ 3 files changed, 85 insertions(+) diff --git a/src/components/piano-roll/PianoRoll.css b/src/components/piano-roll/PianoRoll.css index 6c421d1..84bf294 100644 --- a/src/components/piano-roll/PianoRoll.css +++ b/src/components/piano-roll/PianoRoll.css @@ -314,4 +314,32 @@ align-items: center; justify-content: center; z-index: 100; +} + +.piano-roll-zoom-popup { + position: absolute; + top: 100%; + right: 0; + background-color: #2d2d2d; + border: 1px solid #444; + border-radius: 3px; + padding: 8px 12px; + z-index: 1500; + display: flex; + align-items: center; + gap: 8px; + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.3); + white-space: nowrap; + margin-top: 2px; +} + +.piano-roll-zoom-popup input[type="range"] { + width: 120px; +} + +.piano-roll-zoom-value { + font-size: 12px; + color: #e0e0e0; + min-width: 20px; + text-align: center; } \ No newline at end of file diff --git a/src/components/piano-roll/PianoRoll.tsx b/src/components/piano-roll/PianoRoll.tsx index a63af15..18383a9 100644 --- a/src/components/piano-roll/PianoRoll.tsx +++ b/src/components/piano-roll/PianoRoll.tsx @@ -49,6 +49,9 @@ const PianoRoll: React.FC = ({ // Spectrogram controls (only used in spectrogram mode) const [spectrogramThresholdDb, setSpectrogramThresholdDb] = useState(-25); const [spectrogramPower, setSpectrogramPower] = useState(0.5); + + // Piano roll zoom (1x–8x); updates --region-grid-beat-width CSS variable + const [pianoRollZoom, setPianoRollZoom] = useState(1); // Quantization state const [quantPosition, setQuantPosition] = useState('1/8'); @@ -686,6 +689,17 @@ const PianoRoll: React.FC = ({ container.scrollLeft = clampedScrollLeft; }, [playheadPosition, isPlaying, autoScrollEnabled]); + // Update --region-grid-beat-width when zoom changes; reset on unmount + useEffect(() => { + document.documentElement.style.setProperty('--region-grid-beat-width', `${40 * pianoRollZoom}px`); + if (triggerNoteUpdateRef.current) { + triggerNoteUpdateRef.current(prev => prev + 1); + } + return () => { + document.documentElement.style.setProperty('--region-grid-beat-width', '40px'); + }; + }, [pianoRollZoom]); + // Scroll horizontally to the active region's starting bar useEffect(() => { if (pianoRollContentRef.current && activeRegion) { @@ -927,6 +941,8 @@ const PianoRoll: React.FC = ({ onThresholdChange={setSpectrogramThresholdDb} power={spectrogramPower} onPowerChange={setSpectrogramPower} + zoom={pianoRollZoom} + onZoomChange={setPianoRollZoom} /> void; power?: number; onPowerChange?: (power: number) => void; + zoom: number; + onZoomChange: (value: number) => void; } const PianoRollToolbar: React.FC = ({ @@ -49,11 +51,27 @@ const PianoRollToolbar: React.FC = ({ onThresholdChange, power = 0.5, onPowerChange, + zoom, + onZoomChange, }) => { const isSpectrogram = mode === 'spectrogram'; const showMidiControls = mode !== 'spectrogram'; // midi-edit and hybrid const showSpecControls = mode === 'spectrogram' || mode === 'hybrid'; + const [showZoomSlider, setShowZoomSlider] = React.useState(false); + const zoomSliderRef = React.useRef(null); + + React.useEffect(() => { + if (!showZoomSlider) return; + const handleClickOutside = (e: MouseEvent) => { + if (zoomSliderRef.current && !zoomSliderRef.current.contains(e.target as Node)) { + setShowZoomSlider(false); + } + }; + document.addEventListener('mousedown', handleClickOutside); + return () => document.removeEventListener('mousedown', handleClickOutside); + }, [showZoomSlider]); + return (
{showMidiControls && ( @@ -148,6 +166,29 @@ const PianoRollToolbar: React.FC = ({ />
)} + +
+ + {showZoomSlider && ( +
+ onZoomChange(parseInt(e.target.value))} + /> + {zoom}x +
+ )} +
); From 5890a6d0b93adbb3c742b387a100b0cc798159e5 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Fri, 1 May 2026 23:56:12 -0700 Subject: [PATCH 10/14] fix: piano roll spectrogram is not being zoomed correctly; switching from MIDI region to audio region with piano roll open might cause residual notes. --- src/components/piano-roll/PianoGrid.tsx | 3 +++ src/components/piano-roll/PianoRoll.tsx | 25 ++++++++++--------- .../piano-roll/PianoRollContent.tsx | 5 +++- .../piano-roll/SpectrogramCanvas.tsx | 25 ++++++++++++++++--- 4 files changed, 42 insertions(+), 16 deletions(-) diff --git a/src/components/piano-roll/PianoGrid.tsx b/src/components/piano-roll/PianoGrid.tsx index e49d197..7f181fc 100644 --- a/src/components/piano-roll/PianoGrid.tsx +++ b/src/components/piano-roll/PianoGrid.tsx @@ -32,6 +32,7 @@ interface PianoGridProps { bpm?: number; spectrogramThresholdDb?: number; spectrogramPower?: number; + pianoRollZoom?: number; mode?: 'midi-edit' | 'spectrogram' | 'hybrid'; } @@ -60,6 +61,7 @@ const PianoGrid: React.FC = ({ bpm = 120, spectrogramThresholdDb = -25, spectrogramPower = 0.5, + pianoRollZoom = 1, }) => { const [cursorPosition, setCursorPosition] = useState(null); const [isModifierPressed, setIsModifierPressed] = useState(false); @@ -240,6 +242,7 @@ const PianoGrid: React.FC = ({ bpm={bpm} thresholdDb={spectrogramThresholdDb} power={spectrogramPower} + zoom={pianoRollZoom} /> )} diff --git a/src/components/piano-roll/PianoRoll.tsx b/src/components/piano-roll/PianoRoll.tsx index 18383a9..4306aaa 100644 --- a/src/components/piano-roll/PianoRoll.tsx +++ b/src/components/piano-roll/PianoRoll.tsx @@ -157,23 +157,23 @@ const PianoRoll: React.FC = ({ setActiveRegion(null); return; } - - // Find the region in the tracks + + let found: KGMidiRegion | null = null; for (const track of tracks) { - const regions = track.getRegions(); - const region = regions.find(r => r.getId() === regionId); - + const region = track.getRegions().find(r => r.getId() === regionId); if (region && region instanceof KGMidiRegion) { - setActiveRegion(region); - - if (DEBUG_MODE.PIANO_ROLL) { - console.log(`Active region set in PianoRoll: ${region.getId()}`); - console.log(`Region details: name=${region.getName()}, trackId=${region.getTrackId()}, trackIndex=${region.getTrackIndex()}`); - } - + found = region; break; } } + + // Always update — clears stale MIDI region when switching to an audio region + setActiveRegion(found); + + if (found && DEBUG_MODE.PIANO_ROLL) { + console.log(`Active region set in PianoRoll: ${found.getId()}`); + console.log(`Region details: name=${found.getName()}, trackId=${found.getTrackId()}, trackIndex=${found.getTrackIndex()}`); + } }, [regionId, tracks]); // Sync local state with KGPianoRollState on mount @@ -965,6 +965,7 @@ const PianoRoll: React.FC = ({ bpm={bpm} spectrogramThresholdDb={spectrogramThresholdDb} spectrogramPower={spectrogramPower} + pianoRollZoom={pianoRollZoom} />
= ({ @@ -56,6 +57,7 @@ const PianoRollContent: React.FC = ({ bpm = 120, spectrogramThresholdDb = -25, spectrogramPower = 0.5, + pianoRollZoom = 1, }) => { const isSpectrogram = mode === 'spectrogram'; // Get KGCore instance @@ -234,7 +236,7 @@ const PianoRollContent: React.FC = ({ /> ); }); - }, [activeRegion, noteUpdateCounter, resizingNoteId, draggingNoteId, tempNoteStyles, selectedNoteIds, selectionBoxRender, tracks]); + }, [mode, activeRegion, noteUpdateCounter, resizingNoteId, draggingNoteId, tempNoteStyles, selectedNoteIds, selectionBoxRender, tracks]); const recordingNoteOverlays = useMemo(() => { if (!isRecording || !activeRegion || recordingNotes.length === 0) return null; @@ -281,6 +283,7 @@ const PianoRollContent: React.FC = ({ bpm={bpm} spectrogramThresholdDb={spectrogramThresholdDb} spectrogramPower={spectrogramPower} + pianoRollZoom={pianoRollZoom} > {memoizedNotes} {!isSpectrogram && recordingNoteOverlays} diff --git a/src/components/piano-roll/SpectrogramCanvas.tsx b/src/components/piano-roll/SpectrogramCanvas.tsx index 6b4b90f..9aa3ac0 100644 --- a/src/components/piano-roll/SpectrogramCanvas.tsx +++ b/src/components/piano-roll/SpectrogramCanvas.tsx @@ -12,6 +12,7 @@ interface SpectrogramCanvasProps { bpm: number; thresholdDb: number; power: number; + zoom: number; } const PITCH_BINS = 128; @@ -53,6 +54,7 @@ const SpectrogramCanvas: React.FC = ({ bpm, thresholdDb, power, + zoom, }) => { const canvasRef = useRef(null); const [loading, setLoading] = useState(true); @@ -62,6 +64,11 @@ const SpectrogramCanvas: React.FC = ({ const rawResultRef = useRef(null); const sampleRateRef = useRef(44100); const regionDurationRef = useRef(0); + // Natural (1x) canvas pixel width — set after each draw, used to apply zoom as CSS stretch + const naturalWidthRef = useRef(0); + // Always-current zoom without making it a renderSpectrogram dependency + const zoomRef = useRef(zoom); + useEffect(() => { zoomRef.current = zoom; }, [zoom]); const renderSpectrogram = useCallback(( result: SpectrogramResult, @@ -75,13 +82,12 @@ const SpectrogramCanvas: React.FC = ({ const ctx = canvas.getContext('2d'); if (!ctx) return; - const beatWidth = - parseInt(getComputedStyle(document.documentElement).getPropertyValue('--region-grid-beat-width')) || 40; const noteHeight = parseInt(getComputedStyle(document.documentElement).getPropertyValue('--region-piano-key-height')) || 20; const totalBeats = (regionDurationSeconds * bpm) / 60; - const canvasWidth = Math.ceil(totalBeats * beatWidth); + // Always draw at 1x resolution; zoom is applied as CSS width stretch + const canvasWidth = Math.ceil(totalBeats * 40); const canvasHeight = PITCH_BINS * noteHeight; // Convert dB threshold to linear: values below this → black @@ -131,6 +137,11 @@ const SpectrogramCanvas: React.FC = ({ ctx.imageSmoothingEnabled = true; ctx.imageSmoothingQuality = 'high'; ctx.drawImage(offscreen, 0, 0, canvasWidth, canvasHeight); + + // Store natural width and apply current zoom as CSS stretch (no pixel recompute on zoom) + naturalWidthRef.current = canvasWidth; + canvas.style.width = `${canvasWidth * zoomRef.current}px`; + canvas.style.height = `${canvasHeight}px`; }, [bpm]); // Re-render without re-running the worker when threshold or power changes @@ -146,6 +157,14 @@ const SpectrogramCanvas: React.FC = ({ } }, [thresholdDb, power, renderSpectrogram]); + // Zoom changes: stretch width only, pin height to canvas pixel height + useEffect(() => { + if (canvasRef.current && naturalWidthRef.current > 0) { + canvasRef.current.style.width = `${naturalWidthRef.current * zoom}px`; + canvasRef.current.style.height = `${canvasRef.current.height}px`; + } + }, [zoom]); + // Load audio + run worker when the audio region itself changes useEffect(() => { let cancelled = false; From 6b8a4700229f2b2ca47959da9eadf77942923d4c Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Sat, 2 May 2026 00:09:33 -0700 Subject: [PATCH 11/14] fix: move computing spectrogram text to the center of current viewport of piano roll window --- src/components/piano-roll/PianoGrid.tsx | 3 + src/components/piano-roll/PianoRoll.css | 26 +++++- .../piano-roll/PianoRollContent.tsx | 79 +++++++++++-------- .../piano-roll/SpectrogramCanvas.tsx | 43 ++++------ 4 files changed, 89 insertions(+), 62 deletions(-) diff --git a/src/components/piano-roll/PianoGrid.tsx b/src/components/piano-roll/PianoGrid.tsx index 7f181fc..a406497 100644 --- a/src/components/piano-roll/PianoGrid.tsx +++ b/src/components/piano-roll/PianoGrid.tsx @@ -34,6 +34,7 @@ interface PianoGridProps { spectrogramPower?: number; pianoRollZoom?: number; mode?: 'midi-edit' | 'spectrogram' | 'hybrid'; + onSpectrogramLoadingChange?: (loading: boolean) => void; } interface CursorPosition { @@ -62,6 +63,7 @@ const PianoGrid: React.FC = ({ spectrogramThresholdDb = -25, spectrogramPower = 0.5, pianoRollZoom = 1, + onSpectrogramLoadingChange, }) => { const [cursorPosition, setCursorPosition] = useState(null); const [isModifierPressed, setIsModifierPressed] = useState(false); @@ -243,6 +245,7 @@ const PianoGrid: React.FC = ({ thresholdDb={spectrogramThresholdDb} power={spectrogramPower} zoom={pianoRollZoom} + onLoadingChange={onSpectrogramLoadingChange} /> )} diff --git a/src/components/piano-roll/PianoRoll.css b/src/components/piano-roll/PianoRoll.css index 84bf294..6931e83 100644 --- a/src/components/piano-roll/PianoRoll.css +++ b/src/components/piano-roll/PianoRoll.css @@ -159,13 +159,37 @@ color: #ff4444; } +.piano-roll-content-outer { + position: relative; + flex: 1; + overflow: hidden; +} + .piano-roll-content { display: flex; flex-direction: column; - flex: 1; + height: 100%; overflow: auto; } +.spectrogram-loading-overlay { + position: absolute; + inset: 0; + display: flex; + align-items: center; + justify-content: center; + pointer-events: none; + z-index: 100; +} + +.spectrogram-loading-label { + padding: 6px 10px; + background: rgba(0, 0, 0, 0.6); + color: #aaa; + font-size: 11px; + border-radius: 3px; +} + .piano-grid-header { display: grid; grid-template-columns: repeat(var(--max-number-of-bars), var(--region-grid-bar-width)); diff --git a/src/components/piano-roll/PianoRollContent.tsx b/src/components/piano-roll/PianoRollContent.tsx index e46fd4e..e0858dd 100644 --- a/src/components/piano-roll/PianoRollContent.tsx +++ b/src/components/piano-roll/PianoRollContent.tsx @@ -1,4 +1,4 @@ -import React, { useMemo, useState, useRef, useEffect } from 'react'; +import React, { useMemo, useState, useRef, useEffect, useCallback } from 'react'; import { DEBUG_MODE } from '../../constants'; import { KGMidiRegion } from '../../core/region/KGMidiRegion'; import { KGMidiNote } from '../../core/midi/KGMidiNote'; @@ -60,6 +60,11 @@ const PianoRollContent: React.FC = ({ pianoRollZoom = 1, }) => { const isSpectrogram = mode === 'spectrogram'; + const [spectrogramLoading, setSpectrogramLoading] = useState(false); + const handleSpectrogramLoadingChange = useCallback((loading: boolean) => { + setSpectrogramLoading(loading); + }, []); + // Get KGCore instance const core = KGCore.instance(); @@ -257,38 +262,48 @@ const PianoRollContent: React.FC = ({ }, [isRecording, recordingNotes, activeRegion]); return ( -
- - -
- - - {} : handleGridDoubleClick} - onClick={isSpectrogram ? () => {} : handleCombinedClick} - onMouseDown={isSpectrogram ? () => {} : handleBackgroundMouseDown} - isBoxSelecting={isSpectrogram ? false : isBoxSelectingRef.current} - selectionBox={isSpectrogram ? { startX: 0, startY: 0, endX: 0, endY: 0 } : selectionBoxRef.current} - regionStartBeat={activeRegion?.getStartFromBeat() || 0} - selectedMode={selectedMode} - keySignature={keySignature} - chordGuide={chordGuide} - audioRegion={audioRegion} - trackId={trackId} - projectName={projectName} - bpm={bpm} - spectrogramThresholdDb={spectrogramThresholdDb} - spectrogramPower={spectrogramPower} - pianoRollZoom={pianoRollZoom} - > - {memoizedNotes} - {!isSpectrogram && recordingNoteOverlays} - +
+
+ + +
+ + + {} : handleGridDoubleClick} + onClick={isSpectrogram ? () => {} : handleCombinedClick} + onMouseDown={isSpectrogram ? () => {} : handleBackgroundMouseDown} + isBoxSelecting={isSpectrogram ? false : isBoxSelectingRef.current} + selectionBox={isSpectrogram ? { startX: 0, startY: 0, endX: 0, endY: 0 } : selectionBoxRef.current} + regionStartBeat={activeRegion?.getStartFromBeat() || 0} + selectedMode={selectedMode} + keySignature={keySignature} + chordGuide={chordGuide} + audioRegion={audioRegion} + trackId={trackId} + projectName={projectName} + bpm={bpm} + spectrogramThresholdDb={spectrogramThresholdDb} + spectrogramPower={spectrogramPower} + pianoRollZoom={pianoRollZoom} + onSpectrogramLoadingChange={handleSpectrogramLoadingChange} + > + {memoizedNotes} + {!isSpectrogram && recordingNoteOverlays} + +
+ {spectrogramLoading && ( +
+
+ Computing spectrogram… +
+
+ )}
); }; diff --git a/src/components/piano-roll/SpectrogramCanvas.tsx b/src/components/piano-roll/SpectrogramCanvas.tsx index 9aa3ac0..0675b18 100644 --- a/src/components/piano-roll/SpectrogramCanvas.tsx +++ b/src/components/piano-roll/SpectrogramCanvas.tsx @@ -13,6 +13,7 @@ interface SpectrogramCanvasProps { thresholdDb: number; power: number; zoom: number; + onLoadingChange?: (loading: boolean) => void; } const PITCH_BINS = 128; @@ -55,6 +56,7 @@ const SpectrogramCanvas: React.FC = ({ thresholdDb, power, zoom, + onLoadingChange, }) => { const canvasRef = useRef(null); const [loading, setLoading] = useState(true); @@ -70,6 +72,8 @@ const SpectrogramCanvas: React.FC = ({ const zoomRef = useRef(zoom); useEffect(() => { zoomRef.current = zoom; }, [zoom]); + useEffect(() => { onLoadingChange?.(loading); }, [loading, onLoadingChange]); + const renderSpectrogram = useCallback(( result: SpectrogramResult, sampleRate: number, @@ -240,35 +244,16 @@ const SpectrogramCanvas: React.FC = ({ }, [audioRegion, trackId, projectName, bpm]); return ( - <> - - {loading && ( -
- Computing spectrogram… -
- )} - + ); }; From da8b9496dff9745cfa6f045e030b09c5e9cba1b1 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Sat, 2 May 2026 00:25:41 -0700 Subject: [PATCH 12/14] feat: add cross-component playhead scroll synchronization --- src/components/MainContent.css | 8 ++-- src/components/MainContent.tsx | 41 +++++++++++++++++-- src/components/piano-roll/PianoGridHeader.tsx | 5 ++- src/components/piano-roll/PianoRoll.tsx | 30 +++++++++++++- src/components/track/Track.css | 2 +- src/stores/projectStore.ts | 18 ++++++++ src/styles/variables.css | 1 + 7 files changed, 94 insertions(+), 11 deletions(-) diff --git a/src/components/MainContent.css b/src/components/MainContent.css index 31bc118..2357b90 100644 --- a/src/components/MainContent.css +++ b/src/components/MainContent.css @@ -10,7 +10,7 @@ .main-content-wrapper { display: flex; flex-direction: column; - min-width: calc(200px + var(--max-number-of-bars) * var(--track-grid-bar-width)); /* info width + grid width */ + min-width: calc(var(--track-info-panel-width) + var(--max-number-of-bars) * var(--track-grid-bar-width)); /* info width + grid width */ min-height: fit-content; position: relative; } @@ -19,7 +19,7 @@ position: fixed; top: 50px; left: 0; - width: 200px; + width: var(--track-info-panel-width); height: 20px; background-color: #2d2d2d; border-bottom: 1px solid #3a3a3a; @@ -42,7 +42,7 @@ border-bottom: 1px solid #3a3a3a; background-color: #2d2d2d; z-index: 20; - margin-left: 200px; /* Offset for info-container */ + margin-left: var(--track-info-panel-width); /* Offset for info-container */ width: calc(var(--max-number-of-bars) * var(--track-grid-bar-width)); /* Exact width for 32 bars */ cursor: pointer; /* Show pointer cursor on hover to indicate interactivity */ } @@ -80,7 +80,7 @@ .info-container { position: sticky; left: 0; - width: 200px; + width: var(--track-info-panel-width); z-index: 1002; background-color: #2d2d2d; align-self: flex-start; diff --git a/src/components/MainContent.tsx b/src/components/MainContent.tsx index 0d9e6eb..28eefc5 100644 --- a/src/components/MainContent.tsx +++ b/src/components/MainContent.tsx @@ -50,6 +50,8 @@ const MainContent: React.FC = ({ addAudioTrack, projectName, savedProjectName, + requestPianoRollScroll, + mainContentScrollRequest, } = useProjectStore(); // State to store regions @@ -136,8 +138,10 @@ const MainContent: React.FC = ({ ) || 40; const playheadPixel = barPosition * barWidth; - // Center the playhead in the visible grid area (excluding the 200px sticky info panel) - const infoWidth = 200; + // Center the playhead in the visible grid area (excluding the sticky info panel) + const infoWidth = parseInt( + getComputedStyle(document.documentElement).getPropertyValue('--track-info-panel-width') + ) || 200; const targetScrollLeft = playheadPixel - (container.clientWidth - infoWidth) / 2; const clampedScrollLeft = Math.max( 0, @@ -148,6 +152,36 @@ const MainContent: React.FC = ({ container.scrollLeft = clampedScrollLeft; }, [playheadPosition, isPlaying, autoScrollEnabled, timeSignature]); + // Handle scroll requests from piano roll header clicks + useEffect(() => { + if (mainContentScrollRequest === null) return; + + const container = mainContentRef.current; + if (!container) return; + + const beatsPerBar = timeSignature.numerator; + const barPosition = mainContentScrollRequest / beatsPerBar; + const barWidth = parseInt( + getComputedStyle(document.documentElement).getPropertyValue('--track-grid-bar-width') + ) || 40; + const playheadPixel = barPosition * barWidth; + + // Center the playhead in the visible grid area (excluding the sticky info panel) + const infoWidth = parseInt( + getComputedStyle(document.documentElement).getPropertyValue('--track-info-panel-width') + ) || 200; + const targetScrollLeft = playheadPixel - (container.clientWidth - infoWidth) / 2; + const clampedScrollLeft = Math.max( + 0, + Math.min(targetScrollLeft, container.scrollWidth - container.clientWidth) + ); + + container.scrollLeft = clampedScrollLeft; + + // Clear the request after handling + useProjectStore.setState({ mainContentScrollRequest: null }); + }, [mainContentScrollRequest, timeSignature]); + // Effect to verify track updates useEffect(() => { // Check for pending updates @@ -774,6 +808,7 @@ const MainContent: React.FC = ({ const clickPosition = calculatePlayheadFromMouse(e.clientX); if (clickPosition !== null) { setPlayheadPosition(clickPosition); + requestPianoRollScroll(clickPosition); if (DEBUG_MODE.MAIN_CONTENT) { console.log(`Single click on bar numbers - Set playhead to: ${clickPosition}`); @@ -799,7 +834,7 @@ const MainContent: React.FC = ({ document.removeEventListener('mousemove', handleMouseMove); document.removeEventListener('mouseup', handleMouseUp); }; - }, [calculateBarIndexFromMouse, calculatePlayheadFromMouse, setPlayheadPosition]); + }, [calculateBarIndexFromMouse, calculatePlayheadFromMouse, setPlayheadPosition, requestPianoRollScroll]); const { showInstrumentSelection, isLooping, loopingRange } = useProjectStore(); diff --git a/src/components/piano-roll/PianoGridHeader.tsx b/src/components/piano-roll/PianoGridHeader.tsx index ddd2827..441b8df 100644 --- a/src/components/piano-roll/PianoGridHeader.tsx +++ b/src/components/piano-roll/PianoGridHeader.tsx @@ -14,7 +14,7 @@ const PianoGridHeader: React.FC = ({ timeSignature = { numerator: 4, denominator: 4 } // Default to 4/4 if not provided }) => { // Get store access for playhead position updates - const { setPlayheadPosition } = useProjectStore(); + const { setPlayheadPosition, requestMainContentScroll } = useProjectStore(); // Refs for drag functionality const isDraggingRef = useRef(false); @@ -122,8 +122,9 @@ const PianoGridHeader: React.FC = ({ console.log(`Current bar: ${currentBarNumber} (beat ${currentPlayheadPosition})`); console.log(`Destination bar: ${destinationBarNumber} (beat ${newPosition})`); } - + setPlayheadPosition(newPosition); + requestMainContentScroll(newPosition); } }; diff --git a/src/components/piano-roll/PianoRoll.tsx b/src/components/piano-roll/PianoRoll.tsx index 4306aaa..1f53d01 100644 --- a/src/components/piano-roll/PianoRoll.tsx +++ b/src/components/piano-roll/PianoRoll.tsx @@ -41,7 +41,7 @@ const PianoRoll: React.FC = ({ }) => { const isSpectrogram = mode === 'spectrogram'; const isHybrid = mode === 'hybrid'; - const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled, bpm } = useProjectStore(); + const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled, bpm, pianoRollScrollRequest } = useProjectStore(); // Tool state for piano roll const [activeTool, setActiveTool] = useState<'pointer' | 'pencil'>('pointer'); @@ -689,6 +689,34 @@ const PianoRoll: React.FC = ({ container.scrollLeft = clampedScrollLeft; }, [playheadPosition, isPlaying, autoScrollEnabled]); + // Handle scroll requests from main content bar numbers clicks + useEffect(() => { + if (pianoRollScrollRequest === null) return; + + const container = pianoRollContentRef.current; + if (!container) return; + + const beatWidth = parseInt( + getComputedStyle(document.documentElement).getPropertyValue('--region-grid-beat-width') + ) || 40; + const playheadPixel = pianoRollScrollRequest * beatWidth; + + // Center the playhead in the visible grid area (excluding the 60px sticky piano keys panel) + const keysWidth = parseInt( + getComputedStyle(document.documentElement).getPropertyValue('--region-piano-key-width') + ) || 60; + const targetScrollLeft = playheadPixel - (container.clientWidth - keysWidth) / 2; + const clampedScrollLeft = Math.max( + 0, + Math.min(targetScrollLeft, container.scrollWidth - container.clientWidth) + ); + + container.scrollLeft = clampedScrollLeft; + + // Clear the request after handling + useProjectStore.setState({ pianoRollScrollRequest: null }); + }, [pianoRollScrollRequest]); + // Update --region-grid-beat-width when zoom changes; reset on unmount useEffect(() => { document.documentElement.style.setProperty('--region-grid-beat-width', `${40 * pianoRollZoom}px`); diff --git a/src/components/track/Track.css b/src/components/track/Track.css index 838f481..073cbbd 100644 --- a/src/components/track/Track.css +++ b/src/components/track/Track.css @@ -49,7 +49,7 @@ } .track-info { - width: 200px; + width: var(--track-info-panel-width); height: 120px; padding: 15px 5px 15px 15px; background-color: #2d2d2d; diff --git a/src/stores/projectStore.ts b/src/stores/projectStore.ts index 836f557..f7159a0 100644 --- a/src/stores/projectStore.ts +++ b/src/stores/projectStore.ts @@ -109,6 +109,12 @@ interface ProjectState { canRedo: boolean; undoDescription: string | null; redoDescription: string | null; + + // Cross-component scroll request state + requestMainContentScroll: (beatPosition: number) => void; + requestPianoRollScroll: (beatPosition: number) => void; + mainContentScrollRequest: number | null; + pianoRollScrollRequest: number | null; // Actions setProjectName: (name: string) => void; @@ -340,6 +346,10 @@ export const useProjectStore = create((set, get) => { recordingNotes: [], recordingOriginalPlayhead: 0, + // Initial cross-component scroll request state + mainContentScrollRequest: null, + pianoRollScrollRequest: null, + // Actions setProjectName: (name: string) => { try { @@ -793,6 +803,14 @@ export const useProjectStore = create((set, get) => { set({ autoScrollEnabled: enabled }); }, + requestMainContentScroll: (beatPosition: number) => { + set({ mainContentScrollRequest: beatPosition }); + }, + + requestPianoRollScroll: (beatPosition: number) => { + set({ pianoRollScrollRequest: beatPosition }); + }, + startPlaying: async () => { await KGCore.instance().startPlaying(); set({ isPlaying: true, autoScrollEnabled: true }); diff --git a/src/styles/variables.css b/src/styles/variables.css index a08337e..8d30d51 100644 --- a/src/styles/variables.css +++ b/src/styles/variables.css @@ -2,6 +2,7 @@ --time-signature-numerator: 4; --max-number-of-bars: 32; --track-grid-bar-width: 40px; + --track-info-panel-width: 200px; --region-piano-key-width: 60px; --region-piano-key-height: 20px; --region-grid-beat-width: 40px; From f132f6bbe2aa23ec6a7e17407fb2ec5971f38766 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Sat, 2 May 2026 13:35:17 -0700 Subject: [PATCH 13/14] feat: preserve the viewport position when zooming in and out --- src/components/MainContent.tsx | 37 ++++++++++++++- src/components/piano-roll/PianoRoll.tsx | 60 ++++++++++++++++++++++--- 2 files changed, 88 insertions(+), 9 deletions(-) diff --git a/src/components/MainContent.tsx b/src/components/MainContent.tsx index 28eefc5..00bc552 100644 --- a/src/components/MainContent.tsx +++ b/src/components/MainContent.tsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect, useRef, useCallback } from 'react'; +import React, { useState, useEffect, useRef, useCallback, useLayoutEffect } from 'react'; import './MainContent.css'; import { createPortal } from 'react-dom'; import { useProjectStore } from '../stores/projectStore'; @@ -10,7 +10,7 @@ import TrackInfoPanel from './track/TrackInfoPanel'; import TrackGridPanel from './track/TrackGridPanel'; import PianoRoll from './piano-roll/PianoRoll'; import type { RegionUI } from './interfaces'; -import { DEBUG_MODE, BAR_NUMBERS_CONSTANTS } from '../constants'; +import { DEBUG_MODE, BAR_NUMBERS_CONSTANTS, TOOLBAR_CONSTANTS } from '../constants'; import { useRegionOperations } from '../hooks/useRegionOperations'; import { regionDeleteManager } from '../util/regionDeleteUtil'; import { KGMainContentState } from '../core/state/KGMainContentState'; @@ -26,6 +26,7 @@ const MainContent: React.FC = ({ const { tracks, maxBars, + barWidthMultiplier, reorderTracks, updateTrack, updateTrackProperties, @@ -96,6 +97,7 @@ const MainContent: React.FC = ({ const mainContentRef = useRef(null); const expectedScrollLeftRef = useRef(-1); const isPlayingRef = useRef(false); + const previousBarWidthMultiplierRef = useRef(barWidthMultiplier); // Refs for bar numbers and loop range drag functionality const barNumbersRef = useRef(null); @@ -109,6 +111,37 @@ const MainContent: React.FC = ({ isPlayingRef.current = isPlaying; }, [isPlaying]); + useLayoutEffect(() => { + const previousMultiplier = previousBarWidthMultiplierRef.current; + if (previousMultiplier === barWidthMultiplier) return; + + previousBarWidthMultiplierRef.current = barWidthMultiplier; + + const container = mainContentRef.current; + if (!container) return; + + const infoWidth = parseInt( + getComputedStyle(document.documentElement).getPropertyValue('--track-info-panel-width') + ) || 200; + const visibleMusicWidth = Math.max(0, container.clientWidth - infoWidth); + const previousBarWidth = TOOLBAR_CONSTANTS.BASE_BAR_WIDTH * previousMultiplier; + const nextBarWidth = TOOLBAR_CONSTANTS.BASE_BAR_WIDTH * barWidthMultiplier; + + if (visibleMusicWidth === 0 || previousBarWidth === 0 || nextBarWidth === 0) return; + + const centerPixelBeforeZoom = container.scrollLeft + visibleMusicWidth / 2; + const anchorBeat = (centerPixelBeforeZoom / previousBarWidth) * timeSignature.numerator; + const targetPixel = (anchorBeat / timeSignature.numerator) * nextBarWidth; + const targetScrollLeft = targetPixel - visibleMusicWidth / 2; + const clampedScrollLeft = Math.max( + 0, + Math.min(targetScrollLeft, container.scrollWidth - container.clientWidth) + ); + + expectedScrollLeftRef.current = clampedScrollLeft; + container.scrollLeft = clampedScrollLeft; + }, [barWidthMultiplier, timeSignature]); + // Detect manual horizontal scroll during playback useEffect(() => { const container = mainContentRef.current; diff --git a/src/components/piano-roll/PianoRoll.tsx b/src/components/piano-roll/PianoRoll.tsx index 1f53d01..144eabf 100644 --- a/src/components/piano-roll/PianoRoll.tsx +++ b/src/components/piano-roll/PianoRoll.tsx @@ -1,11 +1,11 @@ -import React, { useRef, useEffect, useState, useCallback } from 'react'; +import React, { useRef, useEffect, useState, useCallback, useLayoutEffect } from 'react'; import './PianoRoll.css'; import type { MouseEvent } from 'react'; import { useProjectStore } from '../../stores/projectStore'; import { FaGripLines } from 'react-icons/fa'; import { KGMidiRegion } from '../../core/region/KGMidiRegion'; import type { KGAudioRegion } from '../../core/region/KGAudioRegion'; -import { DEBUG_MODE, PIANO_ROLL_CONSTANTS } from '../../constants'; +import { DEBUG_MODE, PIANO_ROLL_CONSTANTS, TOOLBAR_CONSTANTS } from '../../constants'; import PianoRollHeader from './PianoRollHeader'; import PianoRollToolbar from './PianoRollToolbar'; import PianoRollContent from './PianoRollContent'; @@ -81,6 +81,7 @@ const PianoRoll: React.FC = ({ // Refs for auto-scroll during playback const pianoRollExpectedScrollLeftRef = useRef(-1); const pianoRollIsPlayingRef = useRef(false); + const pendingZoomAnchorBeatRef = useRef(null); // Ref for storing the setNoteUpdateCounter function const triggerNoteUpdateRef = useRef> | null>(null); @@ -613,6 +614,30 @@ const PianoRoll: React.FC = ({ } }, [quantizeSelectedNotes, quantizeNoteLength]); + const handleZoomChange = useCallback((nextZoom: number) => { + if (nextZoom === pianoRollZoom) return; + + const container = pianoRollContentRef.current; + pendingZoomAnchorBeatRef.current = null; + if (container) { + const keysWidth = parseInt( + getComputedStyle(document.documentElement).getPropertyValue('--region-piano-key-width') + ) || 60; + const visibleMusicWidth = Math.max(0, container.clientWidth - keysWidth); + const beatWidth = parseInt( + getComputedStyle(document.documentElement).getPropertyValue('--region-grid-beat-width') + ) || TOOLBAR_CONSTANTS.BASE_BAR_WIDTH; + + if (visibleMusicWidth > 0 && beatWidth > 0) { + pendingZoomAnchorBeatRef.current = (container.scrollLeft + visibleMusicWidth / 2) / beatWidth; + } else { + pendingZoomAnchorBeatRef.current = null; + } + } + + setPianoRollZoom(nextZoom); + }, [pianoRollZoom]); + // Calculate C4 position and scroll to it when piano roll opens useEffect(() => { if (pianoRollContentRef.current) { @@ -717,12 +742,33 @@ const PianoRoll: React.FC = ({ useProjectStore.setState({ pianoRollScrollRequest: null }); }, [pianoRollScrollRequest]); - // Update --region-grid-beat-width when zoom changes; reset on unmount - useEffect(() => { - document.documentElement.style.setProperty('--region-grid-beat-width', `${40 * pianoRollZoom}px`); + // Update --region-grid-beat-width when zoom changes and preserve the centered beat position. + useLayoutEffect(() => { + const beatWidth = TOOLBAR_CONSTANTS.BASE_BAR_WIDTH * pianoRollZoom; + document.documentElement.style.setProperty('--region-grid-beat-width', `${beatWidth}px`); if (triggerNoteUpdateRef.current) { triggerNoteUpdateRef.current(prev => prev + 1); } + + const anchorBeat = pendingZoomAnchorBeatRef.current; + const container = pianoRollContentRef.current; + if (anchorBeat !== null && container) { + const keysWidth = parseInt( + getComputedStyle(document.documentElement).getPropertyValue('--region-piano-key-width') + ) || 60; + const visibleMusicWidth = Math.max(0, container.clientWidth - keysWidth); + const targetPixel = anchorBeat * beatWidth; + const targetScrollLeft = targetPixel - visibleMusicWidth / 2; + const clampedScrollLeft = Math.max( + 0, + Math.min(targetScrollLeft, container.scrollWidth - container.clientWidth) + ); + + pianoRollExpectedScrollLeftRef.current = clampedScrollLeft; + container.scrollLeft = clampedScrollLeft; + pendingZoomAnchorBeatRef.current = null; + } + return () => { document.documentElement.style.setProperty('--region-grid-beat-width', '40px'); }; @@ -970,7 +1016,7 @@ const PianoRoll: React.FC = ({ power={spectrogramPower} onPowerChange={setSpectrogramPower} zoom={pianoRollZoom} - onZoomChange={setPianoRollZoom} + onZoomChange={handleZoomChange} /> = ({ ); }; -export default PianoRoll; \ No newline at end of file +export default PianoRoll; From 9828c45050d03847ee853984903cfde1ac005b6b Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Sat, 2 May 2026 16:20:44 -0700 Subject: [PATCH 14/14] docs: updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 52d35d6..4bb79dd 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ K.G.Studio is a lightweight, modern DAW that runs entirely in the browser with * ## Latest Updates +- **2026.05.02**: Added **audio track spectrogram visualization** — audio regions now display a real-time spectrogram overlay in the track grid. Added **Piano Roll hybrid mode**: open the piano roll on a MIDI region while an adjacent audio region's spectrogram is shown as a reference layer, letting you edit MIDI notes against the visual shape of the audio. Added **piano roll zoom in/out** with viewport-position preservation so the view stays anchored to the current playhead. Added **fine-tune region position**: nudge regions by small increments for precise placement. Also added cross-component playhead scroll synchronization so the main grid and piano roll stay in sync during playback. - **2026.04.29**: Added **Remix** and **Repaint** to the K.G.One Music Generator panel (powered by ACE-Step 1.5). **Remix** lets you cover an existing audio region in a new style — select an audio region, describe the target style and optionally provide new lyrics, and ACE-Step will re-perform the song with the prompted instrumentation and feel. **Repaint** lets you surgically re-generate a specific section of a song — set a loop range on the timeline to define the repaint window, then describe what you want that section to sound like; the rest of the song stays untouched. Both tools support the same import workflow as the other K.G.One tabs: preview the result in the built-in player, drag it onto a track, or click **Import Aligned to Source** to automatically place it below the original region in a new track. - **2026.04.24**: Added [**K.G.One Music Studio**](https://github.com/KGAudioLab/K.G.One) integration! When K.G.Studio connects to a local K.G.One server, the **K.G.One Music Generator** panel (magic wand button ✦ in the toolbar) becomes available with three AI-powered tools: **Full Song Generation** (powered by ACE-Step 1.5 — generate full-length songs from text prompts), **Clip Generation** (powered by Foundation-1 — generate instrument clips and MIDI loops from text), and **Stem Separation** (powered by python-audio-separator — split any audio into vocals, instrumentals, and more). Generated audio and MIDI can be previewed instantly and dragged directly onto your tracks. K.G.One runs entirely on your own machine (Windows/Linux, CUDA GPU required); see the [K.G.One repository](https://github.com/KGAudioLab/K.G.One) for setup instructions. - **2026.04.11**: Migrated project storage from IndexedDB to OPFS (Origin Private File System) with a folder-based structure for better media file handling. Added audio track support with WAV/MP3 import, playback, looping, and non-destructive region trimming. Added bounce-to-WAV/MP3 export via offline rendering.