diff --git a/README.md b/README.md index 52d35d6..4bb79dd 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ K.G.Studio is a lightweight, modern DAW that runs entirely in the browser with * ## Latest Updates +- **2026.05.02**: Added **audio track spectrogram visualization** — audio regions now display a real-time spectrogram overlay in the track grid. Added **Piano Roll hybrid mode**: open the piano roll on a MIDI region while an adjacent audio region's spectrogram is shown as a reference layer, letting you edit MIDI notes against the visual shape of the audio. Added **piano roll zoom in/out** with viewport-position preservation so the view stays anchored to the current playhead. Added **fine-tune region position**: nudge regions by small increments for precise placement. Also added cross-component playhead scroll synchronization so the main grid and piano roll stay in sync during playback. - **2026.04.29**: Added **Remix** and **Repaint** to the K.G.One Music Generator panel (powered by ACE-Step 1.5). **Remix** lets you cover an existing audio region in a new style — select an audio region, describe the target style and optionally provide new lyrics, and ACE-Step will re-perform the song with the prompted instrumentation and feel. **Repaint** lets you surgically re-generate a specific section of a song — set a loop range on the timeline to define the repaint window, then describe what you want that section to sound like; the rest of the song stays untouched. Both tools support the same import workflow as the other K.G.One tabs: preview the result in the built-in player, drag it onto a track, or click **Import Aligned to Source** to automatically place it below the original region in a new track. - **2026.04.24**: Added [**K.G.One Music Studio**](https://github.com/KGAudioLab/K.G.One) integration! When K.G.Studio connects to a local K.G.One server, the **K.G.One Music Generator** panel (magic wand button ✦ in the toolbar) becomes available with three AI-powered tools: **Full Song Generation** (powered by ACE-Step 1.5 — generate full-length songs from text prompts), **Clip Generation** (powered by Foundation-1 — generate instrument clips and MIDI loops from text), and **Stem Separation** (powered by python-audio-separator — split any audio into vocals, instrumentals, and more). Generated audio and MIDI can be previewed instantly and dragged directly onto your tracks. K.G.One runs entirely on your own machine (Windows/Linux, CUDA GPU required); see the [K.G.One repository](https://github.com/KGAudioLab/K.G.One) for setup instructions. - **2026.04.11**: Migrated project storage from IndexedDB to OPFS (Origin Private File System) with a folder-based structure for better media file handling. Added audio track support with WAV/MP3 import, playback, looping, and non-destructive region trimming. Added bounce-to-WAV/MP3 export via offline rendering. diff --git a/package-lock.json b/package-lock.json index 5454528..70ced97 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,16 @@ { "name": "K.G.Studio", - "version": "0.10.0-build.20260411", + "version": "0.12.0-build.20260430", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "K.G.Studio", - "version": "0.10.0-build.20260411", + "version": "0.12.0-build.20260430", "dependencies": { "@breezystack/lamejs": "^1.2.7", "class-transformer": "^0.5.1", + "fft.js": "^4.0.4", "idb": "^8.0.3", "jszip": "^3.10.1", "openai": "^6.33.0", @@ -4691,6 +4692,12 @@ "dev": true, "license": "MIT" }, + "node_modules/fft.js": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/fft.js/-/fft.js-4.0.4.tgz", + "integrity": "sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==", + "license": "MIT" + }, "node_modules/figures": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/figures/-/figures-5.0.0.tgz", diff --git a/package.json b/package.json index 90d7bfd..05bdb79 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "dependencies": { "@breezystack/lamejs": "^1.2.7", "class-transformer": "^0.5.1", + "fft.js": "^4.0.4", "idb": "^8.0.3", "jszip": "^3.10.1", "openai": "^6.33.0", diff --git a/src/components/MainContent.css b/src/components/MainContent.css index 31bc118..2357b90 100644 --- a/src/components/MainContent.css +++ b/src/components/MainContent.css @@ -10,7 +10,7 @@ .main-content-wrapper { display: flex; flex-direction: column; - min-width: calc(200px + var(--max-number-of-bars) * var(--track-grid-bar-width)); /* info width + grid width */ + min-width: calc(var(--track-info-panel-width) + var(--max-number-of-bars) * var(--track-grid-bar-width)); /* info width + grid width */ min-height: fit-content; position: relative; } @@ -19,7 +19,7 @@ position: fixed; top: 50px; left: 0; - width: 200px; + width: var(--track-info-panel-width); height: 20px; background-color: #2d2d2d; border-bottom: 1px solid #3a3a3a; @@ -42,7 +42,7 @@ border-bottom: 1px solid #3a3a3a; background-color: #2d2d2d; z-index: 20; - margin-left: 200px; /* Offset for info-container */ + margin-left: var(--track-info-panel-width); /* Offset for info-container */ width: calc(var(--max-number-of-bars) * var(--track-grid-bar-width)); /* Exact width for 32 bars */ cursor: pointer; /* Show pointer cursor on hover to indicate interactivity */ } @@ -80,7 +80,7 @@ .info-container { position: sticky; left: 0; - width: 200px; + width: var(--track-info-panel-width); z-index: 1002; background-color: #2d2d2d; align-self: flex-start; diff --git a/src/components/MainContent.tsx b/src/components/MainContent.tsx index d5cce65..00bc552 100644 --- a/src/components/MainContent.tsx +++ b/src/components/MainContent.tsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect, useRef, useCallback } from 'react'; +import React, { useState, useEffect, useRef, useCallback, useLayoutEffect } from 'react'; import './MainContent.css'; import { createPortal } from 'react-dom'; import { useProjectStore } from '../stores/projectStore'; @@ -10,7 +10,7 @@ import TrackInfoPanel from './track/TrackInfoPanel'; import TrackGridPanel from './track/TrackGridPanel'; import PianoRoll from './piano-roll/PianoRoll'; import type { RegionUI } from './interfaces'; -import { DEBUG_MODE, BAR_NUMBERS_CONSTANTS } from '../constants'; +import { DEBUG_MODE, BAR_NUMBERS_CONSTANTS, TOOLBAR_CONSTANTS } from '../constants'; import { useRegionOperations } from '../hooks/useRegionOperations'; import { regionDeleteManager } from '../util/regionDeleteUtil'; import { KGMainContentState } from '../core/state/KGMainContentState'; @@ -26,6 +26,7 @@ const MainContent: React.FC = ({ const { tracks, maxBars, + barWidthMultiplier, reorderTracks, updateTrack, updateTrackProperties, @@ -41,9 +42,17 @@ const MainContent: React.FC = ({ activeRegionId, setShowPianoRoll, setActiveRegionId, + pianoRollMode, + openMidiPianoRoll, + openSpectrogramViewer, + openHybridMode, + hybridAudioRegionId, addTrack, addAudioTrack, projectName, + savedProjectName, + requestPianoRollScroll, + mainContentScrollRequest, } = useProjectStore(); // State to store regions @@ -88,6 +97,7 @@ const MainContent: React.FC = ({ const mainContentRef = useRef(null); const expectedScrollLeftRef = useRef(-1); const isPlayingRef = useRef(false); + const previousBarWidthMultiplierRef = useRef(barWidthMultiplier); // Refs for bar numbers and loop range drag functionality const barNumbersRef = useRef(null); @@ -101,6 +111,37 @@ const MainContent: React.FC = ({ isPlayingRef.current = isPlaying; }, [isPlaying]); + useLayoutEffect(() => { + const previousMultiplier = previousBarWidthMultiplierRef.current; + if (previousMultiplier === barWidthMultiplier) return; + + previousBarWidthMultiplierRef.current = barWidthMultiplier; + + const container = mainContentRef.current; + if (!container) return; + + const infoWidth = parseInt( + getComputedStyle(document.documentElement).getPropertyValue('--track-info-panel-width') + ) || 200; + const visibleMusicWidth = Math.max(0, container.clientWidth - infoWidth); + const previousBarWidth = TOOLBAR_CONSTANTS.BASE_BAR_WIDTH * previousMultiplier; + const nextBarWidth = TOOLBAR_CONSTANTS.BASE_BAR_WIDTH * barWidthMultiplier; + + if (visibleMusicWidth === 0 || previousBarWidth === 0 || nextBarWidth === 0) return; + + const centerPixelBeforeZoom = container.scrollLeft + visibleMusicWidth / 2; + const anchorBeat = (centerPixelBeforeZoom / previousBarWidth) * timeSignature.numerator; + const targetPixel = (anchorBeat / timeSignature.numerator) * nextBarWidth; + const targetScrollLeft = targetPixel - visibleMusicWidth / 2; + const clampedScrollLeft = Math.max( + 0, + Math.min(targetScrollLeft, container.scrollWidth - container.clientWidth) + ); + + expectedScrollLeftRef.current = clampedScrollLeft; + container.scrollLeft = clampedScrollLeft; + }, [barWidthMultiplier, timeSignature]); + // Detect manual horizontal scroll during playback useEffect(() => { const container = mainContentRef.current; @@ -130,8 +171,10 @@ const MainContent: React.FC = ({ ) || 40; const playheadPixel = barPosition * barWidth; - // Center the playhead in the visible grid area (excluding the 200px sticky info panel) - const infoWidth = 200; + // Center the playhead in the visible grid area (excluding the sticky info panel) + const infoWidth = parseInt( + getComputedStyle(document.documentElement).getPropertyValue('--track-info-panel-width') + ) || 200; const targetScrollLeft = playheadPixel - (container.clientWidth - infoWidth) / 2; const clampedScrollLeft = Math.max( 0, @@ -142,6 +185,36 @@ const MainContent: React.FC = ({ container.scrollLeft = clampedScrollLeft; }, [playheadPosition, isPlaying, autoScrollEnabled, timeSignature]); + // Handle scroll requests from piano roll header clicks + useEffect(() => { + if (mainContentScrollRequest === null) return; + + const container = mainContentRef.current; + if (!container) return; + + const beatsPerBar = timeSignature.numerator; + const barPosition = mainContentScrollRequest / beatsPerBar; + const barWidth = parseInt( + getComputedStyle(document.documentElement).getPropertyValue('--track-grid-bar-width') + ) || 40; + const playheadPixel = barPosition * barWidth; + + // Center the playhead in the visible grid area (excluding the sticky info panel) + const infoWidth = parseInt( + getComputedStyle(document.documentElement).getPropertyValue('--track-info-panel-width') + ) || 200; + const targetScrollLeft = playheadPixel - (container.clientWidth - infoWidth) / 2; + const clampedScrollLeft = Math.max( + 0, + Math.min(targetScrollLeft, container.scrollWidth - container.clientWidth) + ); + + container.scrollLeft = clampedScrollLeft; + + // Clear the request after handling + useProjectStore.setState({ mainContentScrollRequest: null }); + }, [mainContentScrollRequest, timeSignature]); + // Effect to verify track updates useEffect(() => { // Check for pending updates @@ -433,8 +506,9 @@ const MainContent: React.FC = ({ } } - // If piano roll is visible, set this region as the active region - if (showPianoRoll) { + // Keep the active piano roll region in sync when that same region is updated. + // Do not switch the editor to an unrelated region from generic move/resize updates. + if (showPianoRoll && activeRegionId === regionId) { setActiveRegionId(regionId); if (DEBUG_MODE.MAIN_CONTENT) { @@ -508,6 +582,16 @@ const MainContent: React.FC = ({ const track = tracks.find(t => t.getId().toString() === region.trackId); if (!track) return; setSelectedTrack(track.getId().toString()); + + // If the piano roll window is already open, follow the selected region's type + if (showPianoRoll) { + const coreRegion = track.getRegions().find(r => r.getId() === regionId); + if (coreRegion?.getCurrentType() === 'KGAudioRegion') { + openSpectrogramViewer(regionId); + } else if (coreRegion?.getCurrentType() === 'KGMidiRegion') { + openMidiPianoRoll(regionId); + } + } }; // Handle explicit pencil action: select region and open piano roll @@ -528,11 +612,29 @@ const MainContent: React.FC = ({ // Reuse selection logic handleRegionClick(regionId); - // Activate and show piano roll - setActiveRegionId(regionId); - setShowPianoRoll(true); + // Activate and show piano roll in midi-edit mode + openMidiPianoRoll(regionId); }; + // Handle spectrogram viewer open + const handleOpenSpectrogram = (regionId: string) => { + handleRegionClick(regionId); + openSpectrogramViewer(regionId); + }; + + // Handle hybrid mode open (+ button clicked on opposite-type region) + const handleOpenHybrid = (regionId: string) => { + if (pianoRollMode === 'midi-edit' && activeRegionId) { + openHybridMode(activeRegionId, regionId); + } else if (pianoRollMode === 'spectrogram' && activeRegionId) { + openHybridMode(regionId, activeRegionId); + } + }; + + // + button is visible only when piano roll is open and mode is not hybrid + const showHybridButtonForAudio = showPianoRoll && pianoRollMode === 'midi-edit'; + const showHybridButtonForMidi = showPianoRoll && pianoRollMode === 'spectrogram'; + // Handle piano roll close const handlePianoRollClose = () => { setShowPianoRoll(false); @@ -739,6 +841,7 @@ const MainContent: React.FC = ({ const clickPosition = calculatePlayheadFromMouse(e.clientX); if (clickPosition !== null) { setPlayheadPosition(clickPosition); + requestPianoRollScroll(clickPosition); if (DEBUG_MODE.MAIN_CONTENT) { console.log(`Single click on bar numbers - Set playhead to: ${clickPosition}`); @@ -764,7 +867,7 @@ const MainContent: React.FC = ({ document.removeEventListener('mousemove', handleMouseMove); document.removeEventListener('mouseup', handleMouseUp); }; - }, [calculateBarIndexFromMouse, calculatePlayheadFromMouse, setPlayheadPosition]); + }, [calculateBarIndexFromMouse, calculatePlayheadFromMouse, setPlayheadPosition, requestPianoRollScroll]); const { showInstrumentSelection, isLooping, loopingRange } = useProjectStore(); @@ -824,16 +927,49 @@ const MainContent: React.FC = ({ onRegionUpdated={handleRegionUpdated} onRegionClick={handleRegionClick} onOpenPianoRoll={handleOpenPianoRoll} + onOpenSpectrogram={handleOpenSpectrogram} + showHybridButtonForAudio={showHybridButtonForAudio} + showHybridButtonForMidi={showHybridButtonForMidi} + onOpenHybrid={handleOpenHybrid} onExternalDropComplete={handleExternalDropComplete} /> - {/* Piano Roll - render using portal */} + {/* Piano Roll / Spectrogram Viewer - render using portal */} {showPianoRoll && createPortal( { + // spectrogram mode: audio region IS the activeRegionId + // hybrid mode: audio region is hybridAudioRegionId + const audioId = pianoRollMode === 'spectrogram' ? activeRegionId + : pianoRollMode === 'hybrid' ? hybridAudioRegionId + : null; + if (!audioId) return undefined; + for (const track of tracks) { + const region = track.getRegions().find(r => r.getId() === audioId); + if (region && region.getCurrentType() === 'KGAudioRegion') { + return region as unknown as KGAudioRegion; + } + } + return undefined; + })()} + trackId={(() => { + const audioId = pianoRollMode === 'spectrogram' ? activeRegionId + : pianoRollMode === 'hybrid' ? hybridAudioRegionId + : null; + if (!audioId) return undefined; + for (const track of tracks) { + if (track.getRegions().some(r => r.getId() === audioId)) { + return track.getId().toString(); + } + } + return undefined; + })()} + projectName={savedProjectName} />, document.body )} @@ -841,4 +977,4 @@ const MainContent: React.FC = ({ ); }; -export default MainContent; \ No newline at end of file +export default MainContent; diff --git a/src/components/piano-roll/PianoGrid.tsx b/src/components/piano-roll/PianoGrid.tsx index 14e833c..a406497 100644 --- a/src/components/piano-roll/PianoGrid.tsx +++ b/src/components/piano-roll/PianoGrid.tsx @@ -6,6 +6,8 @@ import { isModifierKeyPressed } from '../../util/osUtil'; import { generatePianoGridBackground, getMatchingChordsForPitch } from '../../util/scaleUtil'; import type { KeySignature } from '../../core/KGProject'; import { KGPianoRollState } from '../../core/state/KGPianoRollState'; +import SpectrogramCanvas from './SpectrogramCanvas'; +import type { KGAudioRegion } from '../../core/region/KGAudioRegion'; interface PianoGridProps { gridRef: MutableRefObject; @@ -24,6 +26,15 @@ interface PianoGridProps { selectedMode: string; keySignature: KeySignature; chordGuide: string; + audioRegion?: KGAudioRegion; + trackId?: string; + projectName?: string; + bpm?: number; + spectrogramThresholdDb?: number; + spectrogramPower?: number; + pianoRollZoom?: number; + mode?: 'midi-edit' | 'spectrogram' | 'hybrid'; + onSpectrogramLoadingChange?: (loading: boolean) => void; } interface CursorPosition { @@ -44,7 +55,15 @@ const PianoGrid: React.FC = ({ regionStartBeat = 0, selectedMode, keySignature, - chordGuide + chordGuide, + audioRegion, + trackId, + projectName, + bpm = 120, + spectrogramThresholdDb = -25, + spectrogramPower = 0.5, + pianoRollZoom = 1, + onSpectrogramLoadingChange, }) => { const [cursorPosition, setCursorPosition] = useState(null); const [isModifierPressed, setIsModifierPressed] = useState(false); @@ -209,13 +228,27 @@ const PianoGrid: React.FC = ({
onMouseDown(e)} onMouseMove={handleMouseMove} onMouseLeave={handleMouseLeave} > + {/* Spectrogram layer — rendered at z-index 0, behind all highlights and notes */} + {audioRegion && trackId && projectName && ( + + )} + {/* Cursor Highlights */} {cursorPosition && ( <> diff --git a/src/components/piano-roll/PianoGridHeader.tsx b/src/components/piano-roll/PianoGridHeader.tsx index ddd2827..441b8df 100644 --- a/src/components/piano-roll/PianoGridHeader.tsx +++ b/src/components/piano-roll/PianoGridHeader.tsx @@ -14,7 +14,7 @@ const PianoGridHeader: React.FC = ({ timeSignature = { numerator: 4, denominator: 4 } // Default to 4/4 if not provided }) => { // Get store access for playhead position updates - const { setPlayheadPosition } = useProjectStore(); + const { setPlayheadPosition, requestMainContentScroll } = useProjectStore(); // Refs for drag functionality const isDraggingRef = useRef(false); @@ -122,8 +122,9 @@ const PianoGridHeader: React.FC = ({ console.log(`Current bar: ${currentBarNumber} (beat ${currentPlayheadPosition})`); console.log(`Destination bar: ${destinationBarNumber} (beat ${newPosition})`); } - + setPlayheadPosition(newPosition); + requestMainContentScroll(newPosition); } }; diff --git a/src/components/piano-roll/PianoRoll.css b/src/components/piano-roll/PianoRoll.css index f6d9536..6931e83 100644 --- a/src/components/piano-roll/PianoRoll.css +++ b/src/components/piano-roll/PianoRoll.css @@ -77,10 +77,17 @@ /* Ensure toolbar and its dropdowns appear above piano roll content */ } +.piano-roll-toolbar .tool-button { + width: 20px; + height: 20px; +} + /* Override pointer-events for piano roll toolbar sections */ .piano-roll-toolbar .toolbar-left, .piano-roll-toolbar .toolbar-right { pointer-events: auto; + /* max-width: 70%; */ + width: auto; } .piano-roll-toolbar .quant-button { @@ -96,12 +103,44 @@ left: 0; } +/* Spectrogram toolbar controls */ +.spectrogram-toolbar-controls { + display: flex; + align-items: center; + gap: 6px; + margin-left: 5px; +} + +.spectrogram-control-label { + font-size: 10px; + color: #aaa; + white-space: nowrap; +} + +.spectrogram-threshold-slider { + /* Match the visual width of the Qua. Pos. / Qua. Len. buttons (~80px) */ + width: 80px; + height: 4px; + accent-color: #7a9ccf; + cursor: pointer; + margin: 0; +} + +.spectrogram-threshold-value { + font-size: 10px; + color: #e0e0e0; + min-width: 30px; + text-align: right; + white-space: nowrap; +} + .piano-roll-title { flex: 1; text-align: center; - font-size: 14px; + font-size: 12px; + font-weight: bold; color: #e0e0e0; - text-transform: uppercase; + /* text-transform: uppercase; */ cursor: pointer; padding: 5px; border-radius: 3px; @@ -120,13 +159,37 @@ color: #ff4444; } +.piano-roll-content-outer { + position: relative; + flex: 1; + overflow: hidden; +} + .piano-roll-content { display: flex; flex-direction: column; - flex: 1; + height: 100%; overflow: auto; } +.spectrogram-loading-overlay { + position: absolute; + inset: 0; + display: flex; + align-items: center; + justify-content: center; + pointer-events: none; + z-index: 100; +} + +.spectrogram-loading-label { + padding: 6px 10px; + background: rgba(0, 0, 0, 0.6); + color: #aaa; + font-size: 11px; + border-radius: 3px; +} + .piano-grid-header { display: grid; grid-template-columns: repeat(var(--max-number-of-bars), var(--region-grid-bar-width)); @@ -275,4 +338,32 @@ align-items: center; justify-content: center; z-index: 100; +} + +.piano-roll-zoom-popup { + position: absolute; + top: 100%; + right: 0; + background-color: #2d2d2d; + border: 1px solid #444; + border-radius: 3px; + padding: 8px 12px; + z-index: 1500; + display: flex; + align-items: center; + gap: 8px; + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.3); + white-space: nowrap; + margin-top: 2px; +} + +.piano-roll-zoom-popup input[type="range"] { + width: 120px; +} + +.piano-roll-zoom-value { + font-size: 12px; + color: #e0e0e0; + min-width: 20px; + text-align: center; } \ No newline at end of file diff --git a/src/components/piano-roll/PianoRoll.tsx b/src/components/piano-roll/PianoRoll.tsx index 2fe27a5..144eabf 100644 --- a/src/components/piano-roll/PianoRoll.tsx +++ b/src/components/piano-roll/PianoRoll.tsx @@ -1,10 +1,11 @@ -import React, { useRef, useEffect, useState, useCallback } from 'react'; +import React, { useRef, useEffect, useState, useCallback, useLayoutEffect } from 'react'; import './PianoRoll.css'; import type { MouseEvent } from 'react'; import { useProjectStore } from '../../stores/projectStore'; import { FaGripLines } from 'react-icons/fa'; import { KGMidiRegion } from '../../core/region/KGMidiRegion'; -import { DEBUG_MODE, PIANO_ROLL_CONSTANTS } from '../../constants'; +import type { KGAudioRegion } from '../../core/region/KGAudioRegion'; +import { DEBUG_MODE, PIANO_ROLL_CONSTANTS, TOOLBAR_CONSTANTS } from '../../constants'; import PianoRollHeader from './PianoRollHeader'; import PianoRollToolbar from './PianoRollToolbar'; import PianoRollContent from './PianoRollContent'; @@ -22,18 +23,35 @@ interface PianoRollProps { regionId: string | null; initialPosition?: { x: number; y: number }; initialSize?: { width: number; height: number }; + mode?: 'midi-edit' | 'spectrogram' | 'hybrid'; + audioRegion?: KGAudioRegion; + trackId?: string; + projectName?: string; } const PianoRoll: React.FC = ({ onClose, regionId, initialPosition, - initialSize + initialSize, + mode = 'midi-edit', + audioRegion, + trackId, + projectName, }) => { - const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled } = useProjectStore(); + const isSpectrogram = mode === 'spectrogram'; + const isHybrid = mode === 'hybrid'; + const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled, bpm, pianoRollScrollRequest } = useProjectStore(); // Tool state for piano roll const [activeTool, setActiveTool] = useState<'pointer' | 'pencil'>('pointer'); + + // Spectrogram controls (only used in spectrogram mode) + const [spectrogramThresholdDb, setSpectrogramThresholdDb] = useState(-25); + const [spectrogramPower, setSpectrogramPower] = useState(0.5); + + // Piano roll zoom (1x–8x); updates --region-grid-beat-width CSS variable + const [pianoRollZoom, setPianoRollZoom] = useState(1); // Quantization state const [quantPosition, setQuantPosition] = useState('1/8'); @@ -63,6 +81,7 @@ const PianoRoll: React.FC = ({ // Refs for auto-scroll during playback const pianoRollExpectedScrollLeftRef = useRef(-1); const pianoRollIsPlayingRef = useRef(false); + const pendingZoomAnchorBeatRef = useRef(null); // Ref for storing the setNoteUpdateCounter function const triggerNoteUpdateRef = useRef> | null>(null); @@ -139,23 +158,23 @@ const PianoRoll: React.FC = ({ setActiveRegion(null); return; } - - // Find the region in the tracks + + let found: KGMidiRegion | null = null; for (const track of tracks) { - const regions = track.getRegions(); - const region = regions.find(r => r.getId() === regionId); - + const region = track.getRegions().find(r => r.getId() === regionId); if (region && region instanceof KGMidiRegion) { - setActiveRegion(region); - - if (DEBUG_MODE.PIANO_ROLL) { - console.log(`Active region set in PianoRoll: ${region.getId()}`); - console.log(`Region details: name=${region.getName()}, trackId=${region.getTrackId()}, trackIndex=${region.getTrackIndex()}`); - } - + found = region; break; } } + + // Always update — clears stale MIDI region when switching to an audio region + setActiveRegion(found); + + if (found && DEBUG_MODE.PIANO_ROLL) { + console.log(`Active region set in PianoRoll: ${found.getId()}`); + console.log(`Region details: name=${found.getName()}, trackId=${found.getTrackId()}, trackIndex=${found.getTrackIndex()}`); + } }, [regionId, tracks]); // Sync local state with KGPianoRollState on mount @@ -595,6 +614,30 @@ const PianoRoll: React.FC = ({ } }, [quantizeSelectedNotes, quantizeNoteLength]); + const handleZoomChange = useCallback((nextZoom: number) => { + if (nextZoom === pianoRollZoom) return; + + const container = pianoRollContentRef.current; + pendingZoomAnchorBeatRef.current = null; + if (container) { + const keysWidth = parseInt( + getComputedStyle(document.documentElement).getPropertyValue('--region-piano-key-width') + ) || 60; + const visibleMusicWidth = Math.max(0, container.clientWidth - keysWidth); + const beatWidth = parseInt( + getComputedStyle(document.documentElement).getPropertyValue('--region-grid-beat-width') + ) || TOOLBAR_CONSTANTS.BASE_BAR_WIDTH; + + if (visibleMusicWidth > 0 && beatWidth > 0) { + pendingZoomAnchorBeatRef.current = (container.scrollLeft + visibleMusicWidth / 2) / beatWidth; + } else { + pendingZoomAnchorBeatRef.current = null; + } + } + + setPianoRollZoom(nextZoom); + }, [pianoRollZoom]); + // Calculate C4 position and scroll to it when piano roll opens useEffect(() => { if (pianoRollContentRef.current) { @@ -671,6 +714,66 @@ const PianoRoll: React.FC = ({ container.scrollLeft = clampedScrollLeft; }, [playheadPosition, isPlaying, autoScrollEnabled]); + // Handle scroll requests from main content bar numbers clicks + useEffect(() => { + if (pianoRollScrollRequest === null) return; + + const container = pianoRollContentRef.current; + if (!container) return; + + const beatWidth = parseInt( + getComputedStyle(document.documentElement).getPropertyValue('--region-grid-beat-width') + ) || 40; + const playheadPixel = pianoRollScrollRequest * beatWidth; + + // Center the playhead in the visible grid area (excluding the 60px sticky piano keys panel) + const keysWidth = parseInt( + getComputedStyle(document.documentElement).getPropertyValue('--region-piano-key-width') + ) || 60; + const targetScrollLeft = playheadPixel - (container.clientWidth - keysWidth) / 2; + const clampedScrollLeft = Math.max( + 0, + Math.min(targetScrollLeft, container.scrollWidth - container.clientWidth) + ); + + container.scrollLeft = clampedScrollLeft; + + // Clear the request after handling + useProjectStore.setState({ pianoRollScrollRequest: null }); + }, [pianoRollScrollRequest]); + + // Update --region-grid-beat-width when zoom changes and preserve the centered beat position. + useLayoutEffect(() => { + const beatWidth = TOOLBAR_CONSTANTS.BASE_BAR_WIDTH * pianoRollZoom; + document.documentElement.style.setProperty('--region-grid-beat-width', `${beatWidth}px`); + if (triggerNoteUpdateRef.current) { + triggerNoteUpdateRef.current(prev => prev + 1); + } + + const anchorBeat = pendingZoomAnchorBeatRef.current; + const container = pianoRollContentRef.current; + if (anchorBeat !== null && container) { + const keysWidth = parseInt( + getComputedStyle(document.documentElement).getPropertyValue('--region-piano-key-width') + ) || 60; + const visibleMusicWidth = Math.max(0, container.clientWidth - keysWidth); + const targetPixel = anchorBeat * beatWidth; + const targetScrollLeft = targetPixel - visibleMusicWidth / 2; + const clampedScrollLeft = Math.max( + 0, + Math.min(targetScrollLeft, container.scrollWidth - container.clientWidth) + ); + + pianoRollExpectedScrollLeftRef.current = clampedScrollLeft; + container.scrollLeft = clampedScrollLeft; + pendingZoomAnchorBeatRef.current = null; + } + + return () => { + document.documentElement.style.setProperty('--region-grid-beat-width', '40px'); + }; + }, [pianoRollZoom]); + // Scroll horizontally to the active region's starting bar useEffect(() => { if (pianoRollContentRef.current && activeRegion) { @@ -855,6 +958,12 @@ const PianoRoll: React.FC = ({ // Get the title for the piano roll based on the active region const getPianoRollTitle = () => { + if (isSpectrogram) return audioRegion ? `SPECTROGRAM — ${audioRegion.getName()}` : 'SPECTROGRAM'; + if (isHybrid) { + const midiName = activeRegion?.getName() ?? 'MIDI'; + const audioName = audioRegion?.getName() ?? 'Audio'; + return `${midiName} + ${audioName}`; + } if (!activeRegion) return "EDIT NOTE CLIP"; // Calculate the bar and beat position of the region @@ -901,8 +1010,15 @@ const PianoRoll: React.FC = ({ chordGuide={chordGuide} onChordGuideChange={handleChordGuideSelect} blinkButton={blinkButton} + mode={mode} + thresholdDb={spectrogramThresholdDb} + onThresholdChange={setSpectrogramThresholdDb} + power={spectrogramPower} + onPowerChange={setSpectrogramPower} + zoom={pianoRollZoom} + onZoomChange={handleZoomChange} /> - + = ({ selectedMode={selectedMode} keySignature={keySignature} chordGuide={chordGuide} + mode={mode} + audioRegion={audioRegion} + trackId={trackId} + projectName={projectName} + bpm={bpm} + spectrogramThresholdDb={spectrogramThresholdDb} + spectrogramPower={spectrogramPower} + pianoRollZoom={pianoRollZoom} />
= ({ ); }; -export default PianoRoll; \ No newline at end of file +export default PianoRoll; diff --git a/src/components/piano-roll/PianoRollContent.tsx b/src/components/piano-roll/PianoRollContent.tsx index d669a97..e0858dd 100644 --- a/src/components/piano-roll/PianoRollContent.tsx +++ b/src/components/piano-roll/PianoRollContent.tsx @@ -1,4 +1,4 @@ -import React, { useMemo, useState, useRef, useEffect } from 'react'; +import React, { useMemo, useState, useRef, useEffect, useCallback } from 'react'; import { DEBUG_MODE } from '../../constants'; import { KGMidiRegion } from '../../core/region/KGMidiRegion'; import { KGMidiNote } from '../../core/midi/KGMidiNote'; @@ -12,6 +12,7 @@ import PianoGrid from './PianoGrid'; import { useNoteOperations } from '../../hooks/useNoteOperations'; import { useNoteSelection } from '../../hooks/useNoteSelection'; import type { KeySignature } from '../../core/KGProject'; +import type { KGAudioRegion } from '../../core/region/KGAudioRegion'; interface PianoRollContentProps { contentRef: React.MutableRefObject; @@ -26,6 +27,14 @@ interface PianoRollContentProps { selectedMode: string; keySignature: KeySignature; chordGuide: string; + mode?: 'midi-edit' | 'spectrogram' | 'hybrid'; + audioRegion?: KGAudioRegion; + trackId?: string; + projectName?: string; + bpm?: number; + spectrogramThresholdDb?: number; + spectrogramPower?: number; + pianoRollZoom?: number; } const PianoRollContent: React.FC = ({ @@ -40,8 +49,22 @@ const PianoRollContent: React.FC = ({ onSetDeleteNotesTrigger, selectedMode, keySignature, - chordGuide + chordGuide, + mode = 'midi-edit', + audioRegion, + trackId, + projectName, + bpm = 120, + spectrogramThresholdDb = -25, + spectrogramPower = 0.5, + pianoRollZoom = 1, }) => { + const isSpectrogram = mode === 'spectrogram'; + const [spectrogramLoading, setSpectrogramLoading] = useState(false); + const handleSpectrogramLoadingChange = useCallback((loading: boolean) => { + setSpectrogramLoading(loading); + }, []); + // Get KGCore instance const core = KGCore.instance(); @@ -105,9 +128,8 @@ const PianoRollContent: React.FC = ({ // Combined click handler for both pointer and pencil modes const handleCombinedClick = (e: React.MouseEvent) => { - // Handle selection click (pointer mode) + if (isSpectrogram) return; handleBackgroundClick(e); - // Handle pencil mode note creation handleGridClick(e); }; @@ -144,7 +166,7 @@ const PianoRollContent: React.FC = ({ // Memoize the notes rendering to prevent unnecessary recalculations const memoizedNotes = useMemo(() => { - if (!activeRegion) return null; + if (isSpectrogram || !activeRegion) return null; if (DEBUG_MODE.PIANO_ROLL) { console.log(`Rendering notes for region: ${activeRegion.getId()}`); @@ -219,7 +241,7 @@ const PianoRollContent: React.FC = ({ /> ); }); - }, [activeRegion, noteUpdateCounter, resizingNoteId, draggingNoteId, tempNoteStyles, selectedNoteIds, selectionBoxRender, tracks]); + }, [mode, activeRegion, noteUpdateCounter, resizingNoteId, draggingNoteId, tempNoteStyles, selectedNoteIds, selectionBoxRender, tracks]); const recordingNoteOverlays = useMemo(() => { if (!isRecording || !activeRegion || recordingNotes.length === 0) return null; @@ -240,31 +262,48 @@ const PianoRollContent: React.FC = ({ }, [isRecording, recordingNotes, activeRegion]); return ( -
- - -
- - - - {memoizedNotes} - {recordingNoteOverlays} - +
+
+ + +
+ + + {} : handleGridDoubleClick} + onClick={isSpectrogram ? () => {} : handleCombinedClick} + onMouseDown={isSpectrogram ? () => {} : handleBackgroundMouseDown} + isBoxSelecting={isSpectrogram ? false : isBoxSelectingRef.current} + selectionBox={isSpectrogram ? { startX: 0, startY: 0, endX: 0, endY: 0 } : selectionBoxRef.current} + regionStartBeat={activeRegion?.getStartFromBeat() || 0} + selectedMode={selectedMode} + keySignature={keySignature} + chordGuide={chordGuide} + audioRegion={audioRegion} + trackId={trackId} + projectName={projectName} + bpm={bpm} + spectrogramThresholdDb={spectrogramThresholdDb} + spectrogramPower={spectrogramPower} + pianoRollZoom={pianoRollZoom} + onSpectrogramLoadingChange={handleSpectrogramLoadingChange} + > + {memoizedNotes} + {!isSpectrogram && recordingNoteOverlays} + +
+ {spectrogramLoading && ( +
+
+ Computing spectrogram… +
+
+ )}
); }; diff --git a/src/components/piano-roll/PianoRollHeader.tsx b/src/components/piano-roll/PianoRollHeader.tsx index ba21bc4..53f05b9 100644 --- a/src/components/piano-roll/PianoRollHeader.tsx +++ b/src/components/piano-roll/PianoRollHeader.tsx @@ -19,12 +19,6 @@ const PianoRollHeader: React.FC = ({ className="piano-roll-header" onMouseDown={onMouseDown} > -
= ({ > {title}
+
); }; diff --git a/src/components/piano-roll/PianoRollToolbar.tsx b/src/components/piano-roll/PianoRollToolbar.tsx index ed12ea0..751fff1 100644 --- a/src/components/piano-roll/PianoRollToolbar.tsx +++ b/src/components/piano-roll/PianoRollToolbar.tsx @@ -4,6 +4,13 @@ import { KGDropdown } from '../common'; import { KGPianoRollState } from '../../core/state/KGPianoRollState'; import { KGCore } from '../../core/KGCore'; +const POWER_OPTIONS = [ + { label: 'Linear', value: '1.0' }, + { label: '√ (default)', value: '0.5' }, + { label: 'Mild', value: '0.4' }, + { label: 'Strong', value: '0.3' }, +]; + interface PianoRollToolbarProps { activeTool: 'pointer' | 'pencil'; onToolSelect: (tool: 'pointer' | 'pencil') => void; @@ -17,6 +24,13 @@ interface PianoRollToolbarProps { chordGuide: string; onChordGuideChange: (value: string) => void; blinkButton?: string | null; + mode?: 'midi-edit' | 'spectrogram' | 'hybrid'; + thresholdDb?: number; + onThresholdChange?: (db: number) => void; + power?: number; + onPowerChange?: (power: number) => void; + zoom: number; + onZoomChange: (value: number) => void; } const PianoRollToolbar: React.FC = ({ @@ -31,82 +45,153 @@ const PianoRollToolbar: React.FC = ({ onModeChange, chordGuide, onChordGuideChange, - blinkButton = null + blinkButton = null, + mode = 'midi-edit', + thresholdDb = -25, + onThresholdChange, + power = 0.5, + onPowerChange, + zoom, + onZoomChange, }) => { + const isSpectrogram = mode === 'spectrogram'; + const showMidiControls = mode !== 'spectrogram'; // midi-edit and hybrid + const showSpecControls = mode === 'spectrogram' || mode === 'hybrid'; + + const [showZoomSlider, setShowZoomSlider] = React.useState(false); + const zoomSliderRef = React.useRef(null); + + React.useEffect(() => { + if (!showZoomSlider) return; + const handleClickOutside = (e: MouseEvent) => { + if (zoomSliderRef.current && !zoomSliderRef.current.contains(e.target as Node)) { + setShowZoomSlider(false); + } + }; + document.addEventListener('mousedown', handleClickOutside); + return () => document.removeEventListener('mousedown', handleClickOutside); + }, [showZoomSlider]); + return (
-
- {/* Left section with mode and chord guide dropdowns */} - ({ label: data.name, value: id }))} - value={selectedMode} - onChange={(value) => onModeChange(value)} - label="Mode" - buttonClassName="mode-dropdown" - showValueAsLabel={true} - /> - onChordGuideChange(value)} - label="Chord" - buttonClassName="chord-guide-dropdown" - showValueAsLabel={true} - /> -
- -
- {/* Center section with pointer and pencil tools */} - - -
- + {showMidiControls && ( +
+ + + ({ label: data.name, value: id }))} + value={selectedMode} + onChange={(value) => onModeChange(value)} + label="Mode" + buttonClassName="mode-dropdown" + showValueAsLabel={true} + /> + onChordGuideChange(value)} + label="Chord" + buttonClassName="chord-guide-dropdown" + showValueAsLabel={true} + /> +
+ )} +
- {/* Right section with quantization options */} - onSnappingSelect(value)} - label="Snap" - buttonClassName="snapping" - showValueAsLabel={true} - /> + {showMidiControls && ( + <> + onSnappingSelect(value)} + label="Snap" + buttonClassName="snapping" + showValueAsLabel={true} + /> + onQuantSelect('position', value)} + label="Qua. Pos." + buttonClassName={`quant-position ${blinkButton === 'quant-position' ? 'button-blink' : ''}`} + /> + onQuantSelect('length', value)} + label="Qua. Len." + buttonClassName={`quant-length ${blinkButton === 'quant-length' ? 'button-blink' : ''}`} + /> + + )} - onQuantSelect('position', value)} - label="Qua. Pos." - buttonClassName={`quant-position ${blinkButton === 'quant-position' ? 'button-blink' : ''}`} - /> + {showSpecControls && ( +
+ Floor + onThresholdChange?.(parseInt(e.target.value))} + title={`Noise floor: ${thresholdDb} dB`} + /> + {thresholdDb} dB + onPowerChange?.(parseFloat(v))} + label="Curve" + buttonClassName="curve-dropdown" + showValueAsLabel={true} + /> +
+ )} - onQuantSelect('length', value)} - label="Qua. Len." - buttonClassName={`quant-length ${blinkButton === 'quant-length' ? 'button-blink' : ''}`} - /> +
+ + {showZoomSlider && ( +
+ onZoomChange(parseInt(e.target.value))} + /> + {zoom}x +
+ )} +
); }; -export default PianoRollToolbar; \ No newline at end of file +export default PianoRollToolbar; diff --git a/src/components/piano-roll/SpectrogramCanvas.tsx b/src/components/piano-roll/SpectrogramCanvas.tsx new file mode 100644 index 0000000..0675b18 --- /dev/null +++ b/src/components/piano-roll/SpectrogramCanvas.tsx @@ -0,0 +1,260 @@ +import React, { useCallback, useEffect, useRef, useState } from 'react'; +import * as Tone from 'tone'; +import { KGAudioRegion } from '../../core/region/KGAudioRegion'; +import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface'; +import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage'; +import type { SpectrogramRequest, SpectrogramResult } from '../../workers/spectrogramWorker'; + +interface SpectrogramCanvasProps { + audioRegion: KGAudioRegion; + trackId: string; + projectName: string; + bpm: number; + thresholdDb: number; + power: number; + zoom: number; + onLoadingChange?: (loading: boolean) => void; +} + +const PITCH_BINS = 128; +const HOP_SIZE = 1024; + +// Piecewise-linear RGB colormap: black → dark blue → blue → purple → red → orange → yellow +// Hue rotates 240°→300°→0°→60°, bypassing green entirely. +const COLORMAP_STOPS: Array<[number, [number, number, number]]> = [ + [0.00, [ 0, 0, 0]], + [0.15, [ 0, 0, 180]], + [0.35, [ 0, 60, 255]], + [0.55, [180, 0, 120]], + [0.70, [255, 0, 0]], + [0.85, [255, 140, 0]], + [1.00, [255, 255, 0]], +]; + +function hotColormap(v: number): [number, number, number] { + v = Math.max(0, Math.min(1, v)); + for (let i = 0; i < COLORMAP_STOPS.length - 1; i++) { + const [t0, c0] = COLORMAP_STOPS[i]; + const [t1, c1] = COLORMAP_STOPS[i + 1]; + if (v <= t1) { + const t = (v - t0) / (t1 - t0); + return [ + Math.round(c0[0] + t * (c1[0] - c0[0])), + Math.round(c0[1] + t * (c1[1] - c0[1])), + Math.round(c0[2] + t * (c1[2] - c0[2])), + ]; + } + } + return COLORMAP_STOPS[COLORMAP_STOPS.length - 1][1]; +} + +const SpectrogramCanvas: React.FC = ({ + audioRegion, + trackId, + projectName, + bpm, + thresholdDb, + power, + zoom, + onLoadingChange, +}) => { + const canvasRef = useRef(null); + const [loading, setLoading] = useState(true); + const workerRef = useRef(null); + + // Cache the raw linear result so threshold/power changes re-render without re-running FFT + const rawResultRef = useRef(null); + const sampleRateRef = useRef(44100); + const regionDurationRef = useRef(0); + // Natural (1x) canvas pixel width — set after each draw, used to apply zoom as CSS stretch + const naturalWidthRef = useRef(0); + // Always-current zoom without making it a renderSpectrogram dependency + const zoomRef = useRef(zoom); + useEffect(() => { zoomRef.current = zoom; }, [zoom]); + + useEffect(() => { onLoadingChange?.(loading); }, [loading, onLoadingChange]); + + const renderSpectrogram = useCallback(( + result: SpectrogramResult, + sampleRate: number, + regionDurationSeconds: number, + thresholdDb: number, + power: number, + ) => { + const canvas = canvasRef.current; + if (!canvas) return; + const ctx = canvas.getContext('2d'); + if (!ctx) return; + + const noteHeight = + parseInt(getComputedStyle(document.documentElement).getPropertyValue('--region-piano-key-height')) || 20; + + const totalBeats = (regionDurationSeconds * bpm) / 60; + // Always draw at 1x resolution; zoom is applied as CSS width stretch + const canvasWidth = Math.ceil(totalBeats * 40); + const canvasHeight = PITCH_BINS * noteHeight; + + // Convert dB threshold to linear: values below this → black + const linearThreshold = Math.pow(10, thresholdDb / 20); + + // 1. Paint at natural STFT resolution onto an offscreen canvas (timeSteps × 128). + // Row i = pitchIndex i = pitch (107 − i). Apply threshold then power curve. + const offscreen = document.createElement('canvas'); + offscreen.width = result.timeSteps; + offscreen.height = PITCH_BINS; + const offCtx = offscreen.getContext('2d'); + if (!offCtx) return; + + const imgData = offCtx.createImageData(result.timeSteps, PITCH_BINS); + const pixels = imgData.data; + + for (let row = 0; row < PITCH_BINS; row++) { + const pitch = 107 - row; + for (let col = 0; col < result.timeSteps; col++) { + const idx = (row * result.timeSteps + col) * 4; + pixels[idx + 3] = 255; // always opaque + + if (pitch < 12 || pitch > 107) continue; // outside range → black + + const raw = result.data[col * PITCH_BINS + pitch]; + + // Hard threshold: values below noise floor → 0 (black) + // Re-scale surviving range to [0,1] then apply power curve + const gated = raw < linearThreshold + ? 0 + : Math.pow((raw - linearThreshold) / (1 - linearThreshold), power); + + if (gated <= 0) continue; // stays black + + const [r, g, b] = hotColormap(gated); + pixels[idx] = r; + pixels[idx + 1] = g; + pixels[idx + 2] = b; + } + } + + offCtx.putImageData(imgData, 0, 0); + + // 2. Stretch onto the full canvas — browser bilinear filter smooths between bins. + canvas.width = canvasWidth; + canvas.height = canvasHeight; + ctx.imageSmoothingEnabled = true; + ctx.imageSmoothingQuality = 'high'; + ctx.drawImage(offscreen, 0, 0, canvasWidth, canvasHeight); + + // Store natural width and apply current zoom as CSS stretch (no pixel recompute on zoom) + naturalWidthRef.current = canvasWidth; + canvas.style.width = `${canvasWidth * zoomRef.current}px`; + canvas.style.height = `${canvasHeight}px`; + }, [bpm]); + + // Re-render without re-running the worker when threshold or power changes + useEffect(() => { + if (rawResultRef.current) { + renderSpectrogram( + rawResultRef.current, + sampleRateRef.current, + regionDurationRef.current, + thresholdDb, + power, + ); + } + }, [thresholdDb, power, renderSpectrogram]); + + // Zoom changes: stretch width only, pin height to canvas pixel height + useEffect(() => { + if (canvasRef.current && naturalWidthRef.current > 0) { + canvasRef.current.style.width = `${naturalWidthRef.current * zoom}px`; + canvasRef.current.style.height = `${canvasRef.current.height}px`; + } + }, [zoom]); + + // Load audio + run worker when the audio region itself changes + useEffect(() => { + let cancelled = false; + + const compute = async () => { + setLoading(true); + workerRef.current?.terminate(); + workerRef.current = null; + + try { + let audioBuffer: AudioBuffer | undefined = KGAudioInterface.instance().getAudioBuffer( + trackId, + audioRegion.getAudioFileId() + ); + + if (!audioBuffer) { + const arrayBuffer = await KGAudioFileStorage.loadAudioFile( + projectName, + audioRegion.getAudioFileId() + ); + if (cancelled) return; + const actx = Tone.getContext().rawContext as AudioContext; + audioBuffer = await actx.decodeAudioData(arrayBuffer); + } + + if (cancelled) return; + + const pcm = audioBuffer.getChannelData(0); + const sampleRate = audioBuffer.sampleRate; + const regionDurationSeconds = (audioRegion.getLength() * 60) / bpm; + + const worker = new Worker( + new URL('../../workers/spectrogramWorker.ts', import.meta.url), + { type: 'module' } + ); + workerRef.current = worker; + + worker.onmessage = (e: MessageEvent) => { + if (cancelled) { worker.terminate(); return; } + rawResultRef.current = e.data; + sampleRateRef.current = sampleRate; + regionDurationRef.current = regionDurationSeconds; + renderSpectrogram(e.data, sampleRate, regionDurationSeconds, thresholdDb, power); + setLoading(false); + worker.terminate(); + workerRef.current = null; + }; + + const request: SpectrogramRequest = { + pcm: pcm.slice(0) as Float32Array, + sampleRate, + clipStartOffsetSeconds: audioRegion.getClipStartOffsetSeconds(), + regionDurationSeconds, + bpm, + }; + worker.postMessage(request, [request.pcm.buffer]); + } catch (err) { + if (!cancelled) { + console.error('SpectrogramCanvas: failed to compute spectrogram', err); + setLoading(false); + } + } + }; + + compute(); + + return () => { + cancelled = true; + workerRef.current?.terminate(); + workerRef.current = null; + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [audioRegion, trackId, projectName, bpm]); + + return ( + + ); +}; + +export default SpectrogramCanvas; diff --git a/src/components/track/Region.css b/src/components/track/Region.css index 9175075..ef43a56 100644 --- a/src/components/track/Region.css +++ b/src/components/track/Region.css @@ -97,11 +97,23 @@ background-color: #5a9b6a; } -/* Region pencil trigger inside content */ -.region-pencil-btn { +/* Left-side button cluster inside region-content */ +.region-left-buttons { position: absolute; top: 4px; left: 4px; + display: flex; + flex-direction: row; + align-items: center; + gap: 2px; + z-index: 2; +} + +/* Shared style for all region content buttons */ +.region-pencil-btn, +.region-waveform-btn, +.region-spectrogram-btn, +.region-hybrid-btn { background: rgba(0, 0, 0, 0.25); color: #fff; border: 1px solid rgba(255, 255, 255, 0.2); @@ -109,16 +121,63 @@ padding: 2px; margin: 0; cursor: pointer; - z-index: 2; display: inline-flex; align-items: center; justify-content: center; } -.region-pencil-btn:hover { +.region-pencil-btn:hover, +.region-spectrogram-btn:hover, +.region-hybrid-btn:hover { background: rgba(0, 0, 0, 0.35); } +.region-waveform-btn { + display: none; +} + +.region-waveform-btn:hover { + background: rgba(0, 0, 0, 0.35); +} + +/* Fine-move widget: sits inline in the left button cluster */ +.region-fine-move-widget { + display: flex; + flex-direction: row; + align-items: center; + gap: 2px; +} + +.region-fine-move-btn { + background: rgba(0, 0, 0, 0.25); + color: #fff; + border: 1px solid rgba(255, 255, 255, 0.2); + border-radius: 3px; + padding: 2px; + margin: 0; + cursor: ew-resize; + display: inline-flex; + align-items: center; + justify-content: center; + flex-shrink: 0; +} + +.region-fine-move-btn:hover { + background: rgba(0, 0, 0, 0.35); +} + +.region-fine-move-label { + background: rgba(0, 0, 0, 0.25); + color: #fff; + border: 1px solid rgba(255, 255, 255, 0.2); + border-radius: 3px; + padding: 2px 4px; + font-size: 10px; + white-space: nowrap; + line-height: 1; + user-select: none; +} + /* Instrument dropdown specific styles */ .instrument-dropdown .quant-dropdown { min-width: 80px; diff --git a/src/components/track/RegionItem.test.tsx b/src/components/track/RegionItem.test.tsx new file mode 100644 index 0000000..7172df7 --- /dev/null +++ b/src/components/track/RegionItem.test.tsx @@ -0,0 +1,97 @@ +import React from 'react'; +import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest'; +import { render, fireEvent } from '@testing-library/react'; +import RegionItem from './RegionItem'; +import { KGMidiRegion } from '../../core/region/KGMidiRegion'; +import { KGMainContentState } from '../../core/state/KGMainContentState'; + +vi.mock('../../stores/projectStore', () => ({ + useProjectStore: () => ({ + selectedRegionIds: [], + timeSignature: { numerator: 4, denominator: 4 }, + bpm: 120, + }), +})); + +describe('RegionItem', () => { + beforeAll(() => { + Object.defineProperty(HTMLCanvasElement.prototype, 'getContext', { + value: vi.fn(() => ({ + clearRect: vi.fn(), + fillRect: vi.fn(), + })), + }); + + class ResizeObserverMock { + observe() {} + unobserve() {} + disconnect() {} + } + + vi.stubGlobal('ResizeObserver', ResizeObserverMock); + }); + + beforeEach(() => { + KGMainContentState.instance().setActiveTool('pointer'); + }); + + const renderRegion = (props: Partial> = {}) => { + const midiRegion = new KGMidiRegion('midi-1', 'track-1', 0, 'Test Region', 0, 4); + + return render( + + ); + }; + + it('treats small pointer jitter as a click', () => { + const onClick = vi.fn(); + const onDragStart = vi.fn(); + const onDrag = vi.fn(); + const onDragEnd = vi.fn(); + + const { container } = renderRegion({ onClick, onDragStart, onDrag, onDragEnd }); + const region = container.querySelector('.track-region'); + + expect(region).toBeTruthy(); + + fireEvent.mouseDown(region!, { clientX: 100, clientY: 100 }); + fireEvent.mouseMove(document, { clientX: 102, clientY: 102 }); + fireEvent.mouseUp(document, { clientX: 102, clientY: 102 }); + + expect(onClick).toHaveBeenCalledWith('midi-1'); + expect(onDragStart).not.toHaveBeenCalled(); + expect(onDrag).not.toHaveBeenCalled(); + expect(onDragEnd).not.toHaveBeenCalled(); + }); + + it('starts a drag after crossing the movement threshold', () => { + const onClick = vi.fn(); + const onDragStart = vi.fn(); + const onDrag = vi.fn(); + const onDragEnd = vi.fn(); + + const { container } = renderRegion({ onClick, onDragStart, onDrag, onDragEnd }); + const region = container.querySelector('.track-region'); + + expect(region).toBeTruthy(); + + fireEvent.mouseDown(region!, { clientX: 100, clientY: 100 }); + fireEvent.mouseMove(document, { clientX: 110, clientY: 100 }); + fireEvent.mouseUp(document, { clientX: 110, clientY: 100 }); + + expect(onClick).not.toHaveBeenCalled(); + expect(onDragStart).toHaveBeenCalledWith('midi-1', 100, 100); + expect(onDrag).toHaveBeenCalledWith('midi-1', 10, 0); + expect(onDragEnd).toHaveBeenCalledWith('midi-1'); + }); +}); diff --git a/src/components/track/RegionItem.tsx b/src/components/track/RegionItem.tsx index d6f6934..934a9b2 100644 --- a/src/components/track/RegionItem.tsx +++ b/src/components/track/RegionItem.tsx @@ -1,6 +1,7 @@ import React, { useState, useRef, useEffect } from 'react'; import './Region.css'; -import { FaPencilAlt } from 'react-icons/fa'; +import { FaPencilAlt, FaPlus } from 'react-icons/fa'; +import { MdGraphicEq, MdSwapHoriz } from 'react-icons/md'; import type { ResizeAction } from '../interfaces'; import { REGION_CONSTANTS, DEBUG_MODE } from '../../constants'; import { KGMidiRegion } from '../../core/region/KGMidiRegion'; @@ -8,6 +9,8 @@ import { KGAudioRegion } from '../../core/region/KGAudioRegion'; import { useProjectStore } from '../../stores/projectStore'; import { KGMainContentState } from '../../core/state/KGMainContentState'; +const DRAG_START_THRESHOLD_PX = 4; + interface RegionItemProps { id: string; name: string; @@ -27,6 +30,13 @@ interface RegionItemProps { onClick?: (regionId: string) => void; // Explicit open piano roll action from header pencil icon onOpenPianoRoll?: (regionId: string) => void; + // Open spectrogram viewer for audio regions + onOpenSpectrogram?: (regionId: string) => void; + // Enter hybrid mode (show + when piano roll is open with the opposite region type selected) + showHybridButton?: boolean; + onOpenHybrid?: (regionId: string) => void; + // Fine-move end callback — passes raw (unscaled) mouse pixel delta + onFineMoveEnd?: (regionId: string, rawPixelDelta: number) => void; // MIDI region data for rendering notes midiRegion?: KGMidiRegion; // Audio region data for rendering waveform @@ -49,6 +59,10 @@ const RegionItem: React.FC = ({ onDragEnd, onClick, onOpenPianoRoll, + onOpenSpectrogram, + showHybridButton, + onOpenHybrid, + onFineMoveEnd, midiRegion, audioRegion, audioBuffer @@ -60,11 +74,19 @@ const RegionItem: React.FC = ({ const [resizeEdge, setResizeEdge] = useState('none'); const [isResizing, setIsResizing] = useState(false); const [isDragging, setIsDragging] = useState(false); - const initialMousePosRef = useRef<{x: number, y: number}>({x: 0, y: 0}); + const initialMousePosRef = useRef<{ x: number, y: number }>({ x: 0, y: 0 }); // Use refs to track states for immediate access const isResizingRef = useRef(false); const isDraggingRef = useRef(false); - const hasMovedRef = useRef(false); + const isPendingDragRef = useRef(false); + + // Fine-move state + const [isFineDragging, setIsFineDragging] = useState(false); + const [fineDeltaDisplay, setFineDeltaDisplay] = useState('+0.00'); + const [fineTranslateX, setFineTranslateX] = useState(0); + const isFineDraggingRef = useRef(false); + const fineMouseStartXRef = useRef(0); + const fineRawDeltaRef = useRef(0); // Canvas ref for note visualization const canvasRef = useRef(null); @@ -382,7 +404,7 @@ const RegionItem: React.FC = ({ const activeTool = KGMainContentState.instance().getActiveTool(); if (activeTool === 'pencil') { // Still allow click events to pass through for region selection - if (!hasMovedRef.current && onClick) { + if (onClick) { if (DEBUG_MODE.REGION_ITEM) { console.log(`REGION CLICKED (pencil mode): regionId=${id}`); } @@ -395,7 +417,7 @@ const RegionItem: React.FC = ({ e.preventDefault(); // Reset movement tracking - hasMovedRef.current = false; + isPendingDragRef.current = false; // Store initial mouse position initialMousePosRef.current = { x: e.clientX, y: e.clientY }; @@ -414,21 +436,12 @@ const RegionItem: React.FC = ({ onResizeStart(id, resizeEdge, e.clientX); } } else { - // Start dragging + // Wait for actual pointer movement before promoting this gesture to a drag. if (DEBUG_MODE.REGION_ITEM) { - console.log(`DRAG START: regionId=${id}`); + console.log(`PENDING REGION INTERACTION: regionId=${id}`); } - setIsDragging(true); - isDraggingRef.current = true; - - // Change cursor to grabbing during drag - setCursor('grabbing'); - - // Call the onDragStart callback if provided - if (onDragStart) { - onDragStart(id, e.clientX, e.clientY); - } + isPendingDragRef.current = true; } // Add global event listeners for mouse move and up @@ -438,9 +451,6 @@ const RegionItem: React.FC = ({ // Handle global mouse move for resize or drag const handleGlobalMouseMove = (e: MouseEvent) => { - // Set the hasMovedRef to true as soon as there's movement - hasMovedRef.current = true; - if (isResizingRef.current) { // Handle resize if (DEBUG_MODE.REGION_ITEM) { @@ -454,16 +464,35 @@ const RegionItem: React.FC = ({ if (onResize) { onResize(id, resizeEdge, deltaX); } - } else if (isDraggingRef.current) { + } else if (isDraggingRef.current || isPendingDragRef.current) { + const deltaX = e.clientX - initialMousePosRef.current.x; + const deltaY = e.clientY - initialMousePosRef.current.y; + const movedEnough = Math.hypot(deltaX, deltaY) >= DRAG_START_THRESHOLD_PX; + + if (!isDraggingRef.current) { + if (!movedEnough) { + return; + } + + if (DEBUG_MODE.REGION_ITEM) { + console.log(`DRAG START: regionId=${id}`); + } + + isPendingDragRef.current = false; + setIsDragging(true); + isDraggingRef.current = true; + setCursor('grabbing'); + + if (onDragStart) { + onDragStart(id, initialMousePosRef.current.x, initialMousePosRef.current.y); + } + } + // Handle drag if (DEBUG_MODE.REGION_ITEM) { console.log(`DRAG MOVE: regionId=${id}, trackIndex=${trackIndex}`); } - // Calculate delta from initial position - const deltaX = e.clientX - initialMousePosRef.current.x; - const deltaY = e.clientY - initialMousePosRef.current.y; - // Call the onDrag callback if provided if (onDrag) { onDrag(id, deltaX, deltaY); @@ -502,26 +531,62 @@ const RegionItem: React.FC = ({ if (onDragEnd) { onDragEnd(id); } - - // If there was no movement, treat it as a click - if (!hasMovedRef.current && onClick) { - if (DEBUG_MODE.REGION_ITEM) { - console.log(`REGION CLICKED: regionId=${id}`); - } - onClick(id); + } else if (isPendingDragRef.current && onClick) { + if (DEBUG_MODE.REGION_ITEM) { + console.log(`REGION CLICKED: regionId=${id}`); } + onClick(id); } + isPendingDragRef.current = false; + // Remove global event listeners document.removeEventListener('mousemove', handleGlobalMouseMove); document.removeEventListener('mouseup', handleGlobalMouseUp); }; + // Fine-move handlers + const handleFineMoveGlobalMouseMove = (e: MouseEvent) => { + if (!isFineDraggingRef.current) return; + const rawDelta = e.clientX - fineMouseStartXRef.current; + fineRawDeltaRef.current = rawDelta; + const scaledDelta = rawDelta * REGION_CONSTANTS.FINE_MOVE_SPEED_RATIO; + setFineTranslateX(scaledDelta); + setFineDeltaDisplay(scaledDelta >= 0 ? `+${scaledDelta.toFixed(2)}` : `${scaledDelta.toFixed(2)}`); + }; + + const handleFineMoveGlobalMouseUp = () => { + if (!isFineDraggingRef.current) return; + isFineDraggingRef.current = false; + setIsFineDragging(false); + setFineTranslateX(0); + document.removeEventListener('mousemove', handleFineMoveGlobalMouseMove); + document.removeEventListener('mouseup', handleFineMoveGlobalMouseUp); + if (fineRawDeltaRef.current !== 0 && onFineMoveEnd) { + onFineMoveEnd(id, fineRawDeltaRef.current); + } + }; + + const handleFineMoveMouseDown = (e: React.MouseEvent) => { + e.preventDefault(); + e.stopPropagation(); + if (isFineDraggingRef.current) return; + fineMouseStartXRef.current = e.clientX; + fineRawDeltaRef.current = 0; + isFineDraggingRef.current = true; + setIsFineDragging(true); + setFineDeltaDisplay('+0.00'); + document.addEventListener('mousemove', handleFineMoveGlobalMouseMove); + document.addEventListener('mouseup', handleFineMoveGlobalMouseUp); + }; + // Clean up event listeners on unmount useEffect(() => { return () => { document.removeEventListener('mousemove', handleGlobalMouseMove); document.removeEventListener('mouseup', handleGlobalMouseUp); + document.removeEventListener('mousemove', handleFineMoveGlobalMouseMove); + document.removeEventListener('mouseup', handleFineMoveGlobalMouseUp); }; }, []); @@ -538,7 +603,7 @@ const RegionItem: React.FC = ({
= ({ {name}
- {!audioRegion && ( - - )} +
+ {!audioRegion && ( + + )} + {audioRegion && ( + + )} + {audioRegion && ( + + )} + {showHybridButton && ( + + )} +
+ + {isFineDragging && ( + {fineDeltaDisplay} + )} +
+
); }; -export default RegionItem; \ No newline at end of file +export default RegionItem; diff --git a/src/components/track/Track.css b/src/components/track/Track.css index 838f481..073cbbd 100644 --- a/src/components/track/Track.css +++ b/src/components/track/Track.css @@ -49,7 +49,7 @@ } .track-info { - width: 200px; + width: var(--track-info-panel-width); height: 120px; padding: 15px 5px 15px 15px; background-color: #2d2d2d; diff --git a/src/components/track/TrackGridItem.tsx b/src/components/track/TrackGridItem.tsx index 08ee19b..dbb758c 100644 --- a/src/components/track/TrackGridItem.tsx +++ b/src/components/track/TrackGridItem.tsx @@ -24,8 +24,13 @@ interface TrackGridItemProps { onRegionResizeEnd?: (regionId: string, finalBarNumber: number, finalLength: number) => void; onRegionDrag?: (regionId: string, newBarNumber: number, newTrackIndex: number) => void; onRegionDragEnd?: (regionId: string, finalBarNumber: number, finalTrackIndex: number) => void; + onRegionFineMoveEnd?: (regionId: string, deltaInBars: number) => void; onRegionClick?: (regionId: string) => void; onOpenPianoRoll?: (regionId: string) => void; + onOpenSpectrogram?: (regionId: string) => void; + showHybridButtonForAudio?: boolean; + showHybridButtonForMidi?: boolean; + onOpenHybrid?: (regionId: string) => void; allTracks?: KGTrack[]; // Added to access all tracks for drag operations onKGOneClipDrop?: (e: React.DragEvent, trackIndex: number) => void; } @@ -45,8 +50,13 @@ const TrackGridItem: React.FC = ({ onRegionResizeEnd, onRegionDrag, onRegionDragEnd, + onRegionFineMoveEnd, onRegionClick, onOpenPianoRoll, + onOpenSpectrogram, + showHybridButtonForAudio, + showHybridButtonForMidi, + onOpenHybrid, allTracks, onKGOneClipDrop, }) => { @@ -484,12 +494,27 @@ const TrackGridItem: React.FC = ({ currentDragTop.current = null; currentDragRegion.current = null; + if (finalBarNumber === region.barNumber && finalTrackIndex === region.trackIndex) { + if (DEBUG_MODE.TRACK_GRID_ITEM) { + console.log(`Skipping no-op drag update for region ${regionId}`); + } + return; + } + // Notify parent about drag end with final values if (onRegionDragEnd) { onRegionDragEnd(regionId, finalBarNumber, finalTrackIndex); } }; + // Handle fine-move end — convert raw pixel delta to delta in bars and pass up + const handleRegionFineMoveEnd = (regionId: string, rawPixelDelta: number) => { + const barWidth = containerWidth / maxBars; + if (barWidth <= 0) return; + const deltaInBars = (rawPixelDelta * REGION_CONSTANTS.FINE_MOVE_SPEED_RATIO) / barWidth; + onRegionFineMoveEnd?.(regionId, deltaInBars); + }; + // Handle region click const handleRegionClick = (regionId: string) => { if (DEBUG_MODE.TRACK_GRID_ITEM) { @@ -555,6 +580,7 @@ const TrackGridItem: React.FC = ({ onDragStart={handleRegionDragStart} onDrag={handleRegionDrag} onDragEnd={handleRegionDragEnd} + onFineMoveEnd={handleRegionFineMoveEnd} // Keep onClick for selection-only logic if needed by parent onClick={handleRegionClick} // New explicit pencil action — disabled for audio regions @@ -566,6 +592,11 @@ const TrackGridItem: React.FC = ({ onRegionClick(regionId); } }} + onOpenSpectrogram={audioRegion ? (regionId) => { + onOpenSpectrogram?.(regionId); + } : undefined} + showHybridButton={audioRegion ? showHybridButtonForAudio : showHybridButtonForMidi} + onOpenHybrid={onOpenHybrid} midiRegion={midiRegion} audioRegion={audioRegion} audioBuffer={audioBuffer} @@ -576,4 +607,4 @@ const TrackGridItem: React.FC = ({ ); }; -export default TrackGridItem; \ No newline at end of file +export default TrackGridItem; diff --git a/src/components/track/TrackGridPanel.tsx b/src/components/track/TrackGridPanel.tsx index da64b29..c506142 100644 --- a/src/components/track/TrackGridPanel.tsx +++ b/src/components/track/TrackGridPanel.tsx @@ -30,6 +30,10 @@ interface TrackGridPanelProps { onRegionUpdated?: (regionId: string, updates: Partial, expectedModelUpdates?: { startBeat: number, length: number }) => void; onRegionClick?: (regionId: string) => void; onOpenPianoRoll?: (regionId: string) => void; + onOpenSpectrogram?: (regionId: string) => void; + showHybridButtonForAudio?: boolean; + showHybridButtonForMidi?: boolean; + onOpenHybrid?: (regionId: string) => void; onExternalDropComplete?: (trackIndex: number, regionUI: RegionUI) => void; } @@ -46,6 +50,10 @@ const TrackGridPanel: React.FC = ({ onRegionUpdated, onRegionClick, onOpenPianoRoll, + onOpenSpectrogram, + showHybridButtonForAudio, + showHybridButtonForMidi, + onOpenHybrid, onExternalDropComplete, }) => { const gridContainerRef = useRef(null); @@ -378,6 +386,13 @@ const TrackGridPanel: React.FC = ({ // Find the region const region = regions.find(r => r.id === regionId); if (!region) return; + + if (finalBarNumber === region.barNumber && finalTrackIndex === region.trackIndex) { + if (DEBUG_MODE.TRACK_GRID_PANEL) { + console.log(`Skipping no-op move for region ${regionId}`); + } + return; + } // Get the target track const targetTrack = tracks[finalTrackIndex]; @@ -458,6 +473,44 @@ const TrackGridPanel: React.FC = ({ } }; + // Handle fine-move end — execute MoveRegionCommand with float-precision beat position + const handleRegionFineMoveEnd = (regionId: string, deltaInBars: number) => { + const region = regions.find(r => r.id === regionId); + if (!region) return; + const track = tracks.find(t => t.getId().toString() === region.trackId); + if (!track) return; + const coreRegion = track.getRegions().find(r => r.getId() === regionId); + if (!coreRegion) return; + + const beatsPerBar = timeSignature.numerator; + const newStartFromBeat = Math.max(0, coreRegion.getStartFromBeat() + deltaInBars * beatsPerBar); + if (newStartFromBeat === coreRegion.getStartFromBeat()) return; + + try { + // Use constructor directly (NOT fromBarCoordinates) to preserve float precision + const command = new MoveRegionCommand( + regionId, + newStartFromBeat, + track.getId().toString(), + region.trackIndex + ); + KGCore.instance().executeCommand(command); + + if (DEBUG_MODE.TRACK_GRID_PANEL) { + console.log(`Fine-moved region ${regionId}: startFromBeat=${newStartFromBeat}`); + } + + const newBarNumber = newStartFromBeat / beatsPerBar + 1; + onRegionUpdated?.( + regionId, + { barNumber: newBarNumber, trackId: region.trackId, trackIndex: region.trackIndex }, + { startBeat: newStartFromBeat, length: coreRegion.getLength() } + ); + } catch (error) { + console.error('Error executing fine-move:', error); + } + }; + // Handle region click const handleRegionClick = (regionId: string) => { if (DEBUG_MODE.TRACK_GRID_PANEL) { @@ -649,8 +702,13 @@ const TrackGridPanel: React.FC = ({ onRegionResizeEnd={handleRegionResizeEnd} onRegionDrag={handleRegionDrag} onRegionDragEnd={handleRegionDragEnd} + onRegionFineMoveEnd={handleRegionFineMoveEnd} onRegionClick={handleRegionClick} onOpenPianoRoll={onOpenPianoRoll} + onOpenSpectrogram={onOpenSpectrogram} + showHybridButtonForAudio={showHybridButtonForAudio} + showHybridButtonForMidi={showHybridButtonForMidi} + onOpenHybrid={onOpenHybrid} allTracks={tracks} onKGOneClipDrop={handleExternalDrop} /> @@ -668,4 +726,4 @@ const TrackGridPanel: React.FC = ({ ); }; -export default TrackGridPanel; \ No newline at end of file +export default TrackGridPanel; diff --git a/src/constants/uiConstants.ts b/src/constants/uiConstants.ts index a34225f..5bc9bf5 100644 --- a/src/constants/uiConstants.ts +++ b/src/constants/uiConstants.ts @@ -28,6 +28,8 @@ export const REGION_CONSTANTS = { EDGE_THRESHOLD: 10, // Minimum region length in bars MIN_REGION_LENGTH: 1.0, + // Fine-move speed ratio: mouse pixels to region movement pixels + FINE_MOVE_SPEED_RATIO: 0.25, }; // Piano roll related constants diff --git a/src/core/project-upgrader/upgradeToV7.test.ts b/src/core/project-upgrader/upgradeToV7.test.ts index 6df72bd..3bc2b19 100644 --- a/src/core/project-upgrader/upgradeToV7.test.ts +++ b/src/core/project-upgrader/upgradeToV7.test.ts @@ -41,10 +41,10 @@ describe('upgradeToV7', () => { }); it('clamps values that exceed MAX_TRACK_VOLUME_DB', () => { - // linear > 1.0 would give positive dB; cap at +6 + // linear > 1.0 gives positive dB but 2.0 is still below the +12 dB ceiling const project = makeProject([2.0]); upgradeToV7(project); - expect(project.getTracks()[0].getVolume()).toBe(AUDIO_INTERFACE_CONSTANTS.MAX_TRACK_VOLUME_DB); + expect(project.getTracks()[0].getVolume()).toBeCloseTo(20 * Math.log10(2.0), 5); }); it('bumps project structure version to 7', () => { diff --git a/src/core/track/KGMidiTrack.test.ts b/src/core/track/KGMidiTrack.test.ts index 73092a3..6d5aaec 100644 --- a/src/core/track/KGMidiTrack.test.ts +++ b/src/core/track/KGMidiTrack.test.ts @@ -348,7 +348,7 @@ describe('KGMidiTrack', () => { // Volume outside valid dB range is clamped track.setVolume(10); - expect(track.getVolume()).toBe(6); + expect(track.getVolume()).toBe(10); track.setVolume(-100); expect(track.getVolume()).toBe(-60); @@ -401,4 +401,4 @@ describe('KGMidiTrack', () => { }); }); }); -}); \ No newline at end of file +}); diff --git a/src/stores/projectStore.test.ts b/src/stores/projectStore.test.ts new file mode 100644 index 0000000..a99935c --- /dev/null +++ b/src/stores/projectStore.test.ts @@ -0,0 +1,78 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { act } from '@testing-library/react'; +import { KGMidiTrack } from '../core/track/KGMidiTrack'; + +const mockProject = { + getTimeSignature: () => ({ numerator: 4, denominator: 4 }), + getMaxBars: () => 32, + getBarWidthMultiplier: () => 1, + getTracks: () => [new KGMidiTrack('Track 1', 0, 'acoustic_grand_piano')], + getBpm: () => 120, + getKeySignature: () => 'C major', + getName: () => 'Test Project', + getSelectedMode: () => 'major', + getIsLooping: () => false, + getLoopingRange: () => null, +}; + +const mockCore = { + getCurrentProject: () => mockProject, + setPlayheadUpdateCallback: vi.fn(), + setPlaybackStateChangeCallback: vi.fn(), + getSelectedItems: () => [], + onSelectionChanged: vi.fn(), + canUndo: () => false, + canRedo: () => false, + getUndoDescription: () => '', + getRedoDescription: () => '', + setOnCommandHistoryChanged: vi.fn(), + executeCommand: vi.fn(), + clearSelectedItems: vi.fn(), + getStatus: () => 'Ready', + getPlayheadPosition: () => 0, + getIsPlaying: () => false, +}; + +vi.mock('../core/KGCore', () => ({ + KGCore: { + instance: () => mockCore, + }, +})); + +vi.mock('../core/config/ConfigManager', () => ({ + ConfigManager: { + instance: () => ({ + getIsInitialized: () => true, + get: () => false, + }), + }, +})); + +describe('projectStore piano roll state', () => { + beforeEach(() => { + vi.resetModules(); + }); + + it('clears hybrid state when opening a MIDI region', async () => { + const { useProjectStore } = await import('./projectStore'); + + act(() => { + useProjectStore.getState().openHybridMode('midi-a', 'audio-a'); + }); + + let state = useProjectStore.getState(); + expect(state.pianoRollMode).toBe('hybrid'); + expect(state.activeRegionId).toBe('midi-a'); + expect(state.hybridAudioRegionId).toBe('audio-a'); + + act(() => { + useProjectStore.getState().openMidiPianoRoll('midi-b'); + }); + + state = useProjectStore.getState(); + expect(state.showPianoRoll).toBe(true); + expect(state.pianoRollMode).toBe('midi-edit'); + expect(state.activeRegionId).toBe('midi-b'); + expect(state.hybridAudioRegionId).toBeNull(); + }); +}); diff --git a/src/stores/projectStore.ts b/src/stores/projectStore.ts index 9d1f1a1..f7159a0 100644 --- a/src/stores/projectStore.ts +++ b/src/stores/projectStore.ts @@ -78,6 +78,8 @@ interface ProjectState { // Piano roll state showPianoRoll: boolean; activeRegionId: string | null; + pianoRollMode: 'midi-edit' | 'spectrogram' | 'hybrid'; + hybridAudioRegionId: string | null; // ChatBox state showChatBox: boolean; @@ -107,6 +109,12 @@ interface ProjectState { canRedo: boolean; undoDescription: string | null; redoDescription: string | null; + + // Cross-component scroll request state + requestMainContentScroll: (beatPosition: number) => void; + requestPianoRollScroll: (beatPosition: number) => void; + mainContentScrollRequest: number | null; + pianoRollScrollRequest: number | null; // Actions setProjectName: (name: string) => void; @@ -146,6 +154,9 @@ interface ProjectState { // Piano roll actions setShowPianoRoll: (show: boolean) => void; setActiveRegionId: (regionId: string | null) => void; + openMidiPianoRoll: (regionId: string) => void; + openSpectrogramViewer: (regionId: string) => void; + openHybridMode: (midiRegionId: string, audioRegionId: string) => void; // Project state cleanup cleanupProjectState: () => void; @@ -304,6 +315,8 @@ export const useProjectStore = create((set, get) => { // Initial piano roll state showPianoRoll: false, activeRegionId: null, + pianoRollMode: 'midi-edit' as const, + hybridAudioRegionId: null, // Initial ChatBox state showChatBox: initialChatBoxState, @@ -333,6 +346,10 @@ export const useProjectStore = create((set, get) => { recordingNotes: [], recordingOriginalPlayhead: 0, + // Initial cross-component scroll request state + mainContentScrollRequest: null, + pianoRollScrollRequest: null, + // Actions setProjectName: (name: string) => { try { @@ -786,6 +803,14 @@ export const useProjectStore = create((set, get) => { set({ autoScrollEnabled: enabled }); }, + requestMainContentScroll: (beatPosition: number) => { + set({ mainContentScrollRequest: beatPosition }); + }, + + requestPianoRollScroll: (beatPosition: number) => { + set({ pianoRollScrollRequest: beatPosition }); + }, + startPlaying: async () => { await KGCore.instance().startPlaying(); set({ isPlaying: true, autoScrollEnabled: true }); @@ -1028,18 +1053,30 @@ export const useProjectStore = create((set, get) => { setShowPianoRoll: (show: boolean) => { set({ showPianoRoll: show }); }, - + setActiveRegionId: (regionId: string | null) => { set({ activeRegionId: regionId }); }, + + openMidiPianoRoll: (regionId: string) => { + set({ showPianoRoll: true, activeRegionId: regionId, pianoRollMode: 'midi-edit', hybridAudioRegionId: null }); + }, + + openSpectrogramViewer: (regionId: string) => { + set({ showPianoRoll: true, activeRegionId: regionId, pianoRollMode: 'spectrogram', hybridAudioRegionId: null }); + }, + + openHybridMode: (midiRegionId: string, audioRegionId: string) => { + set({ showPianoRoll: true, activeRegionId: midiRegionId, hybridAudioRegionId: audioRegionId, pianoRollMode: 'hybrid' }); + }, // Project state cleanup - used when starting new/loading projects cleanupProjectState: () => { // Close piano roll if it's visible set({ showPianoRoll: false }); - - // Clear active region - set({ activeRegionId: null }); + + // Clear active region and hybrid state + set({ activeRegionId: null, hybridAudioRegionId: null, pianoRollMode: 'midi-edit' }); // Clear any selected items KGCore.instance().clearSelectedItems(); diff --git a/src/styles/variables.css b/src/styles/variables.css index a08337e..8d30d51 100644 --- a/src/styles/variables.css +++ b/src/styles/variables.css @@ -2,6 +2,7 @@ --time-signature-numerator: 4; --max-number-of-bars: 32; --track-grid-bar-width: 40px; + --track-info-panel-width: 200px; --region-piano-key-width: 60px; --region-piano-key-height: 20px; --region-grid-beat-width: 40px; diff --git a/src/workers/spectrogramWorker.ts b/src/workers/spectrogramWorker.ts new file mode 100644 index 0000000..a3a914f --- /dev/null +++ b/src/workers/spectrogramWorker.ts @@ -0,0 +1,103 @@ +import FFT from 'fft.js'; + +type WorkerScopeLike = typeof globalThis & { + onmessage: ((event: MessageEvent) => void) | null; + postMessage: (message: SpectrogramResult, transfer: Transferable[]) => void; +}; + +const workerScope = self as WorkerScopeLike; + +export interface SpectrogramRequest { + pcm: Float32Array; + sampleRate: number; + clipStartOffsetSeconds: number; + regionDurationSeconds: number; + bpm: number; +} + +export interface SpectrogramResult { + data: Float32Array; // [timeSteps × 128] row-major, row = time step, col = pitch 0-127 + timeSteps: number; +} + +const FFT_SIZE = 8192; +const HOP_SIZE = 1024; +const PITCH_BINS = 128; + +function hannWindow(size: number): Float32Array { + const w = new Float32Array(size); + for (let i = 0; i < size; i++) { + w[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / (size - 1))); + } + return w; +} + +workerScope.onmessage = (e: MessageEvent) => { + const { pcm, sampleRate, clipStartOffsetSeconds, regionDurationSeconds } = e.data; + + const startSample = Math.floor(clipStartOffsetSeconds * sampleRate); + const endSample = Math.min(pcm.length, startSample + Math.ceil(regionDurationSeconds * sampleRate)); + const regionSamples = pcm.subarray(startSample, endSample); + + const fft = new FFT(FFT_SIZE); + const hann = hannWindow(FFT_SIZE); + const complexOut = fft.createComplexArray() as number[]; + const inputPadded = new Float32Array(FFT_SIZE); + + const totalHops = Math.max(1, Math.ceil((regionSamples.length - FFT_SIZE) / HOP_SIZE) + 1); + const result = new Float32Array(totalHops * PITCH_BINS); + + let maxVal = 0; + + for (let hop = 0; hop < totalHops; hop++) { + const offset = hop * HOP_SIZE; + + // Fill windowed frame (zero-pad at end if needed) + inputPadded.fill(0); + const available = Math.min(FFT_SIZE, regionSamples.length - offset); + for (let i = 0; i < available; i++) { + inputPadded[i] = regionSamples[offset + i] * hann[i]; + } + + fft.realTransform(complexOut, inputPadded as unknown as number[]); + fft.completeSpectrum(complexOut); + + // Max-pool magnitude into pitch bins: keep the loudest FFT bin per semitone, + // rather than summing. Summing inflates every bin by how many FFT bins land there. + const pitchRow = hop * PITCH_BINS; + const numBins = FFT_SIZE / 2; + + for (let bin = 1; bin < numBins; bin++) { + const freq = (bin * sampleRate) / FFT_SIZE; + if (freq < 20 || freq > 20000) continue; + + // Convert frequency to MIDI pitch; clamp to piano roll range C0–B7 (MIDI 12–107) + const pitch = Math.round(69 + 12 * Math.log2(freq / 440)); + if (pitch < 12 || pitch > 107) continue; + + const re = complexOut[2 * bin]; + const im = complexOut[2 * bin + 1]; + const magnitude = Math.sqrt(re * re + im * im); + + if (magnitude > result[pitchRow + pitch]) { + result[pitchRow + pitch] = magnitude; + } + } + + // Track global max for normalization + for (let p = 0; p < PITCH_BINS; p++) { + if (result[pitchRow + p] > maxVal) maxVal = result[pitchRow + p]; + } + } + + // Linear normalization only — threshold and power curve are applied in the canvas + // renderer so changing them is instant (no need to re-run the FFT). + if (maxVal > 0) { + for (let i = 0; i < result.length; i++) { + result[i] = result[i] / maxVal; + } + } + + const response: SpectrogramResult = { data: result, timeSteps: totalHops }; + workerScope.postMessage(response, [result.buffer]); +};