diff --git a/package-lock.json b/package-lock.json index 5454528..70ced97 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,16 @@ { "name": "K.G.Studio", - "version": "0.10.0-build.20260411", + "version": "0.12.0-build.20260430", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "K.G.Studio", - "version": "0.10.0-build.20260411", + "version": "0.12.0-build.20260430", "dependencies": { "@breezystack/lamejs": "^1.2.7", "class-transformer": "^0.5.1", + "fft.js": "^4.0.4", "idb": "^8.0.3", "jszip": "^3.10.1", "openai": "^6.33.0", @@ -4691,6 +4692,12 @@ "dev": true, "license": "MIT" }, + "node_modules/fft.js": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/fft.js/-/fft.js-4.0.4.tgz", + "integrity": "sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==", + "license": "MIT" + }, "node_modules/figures": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/figures/-/figures-5.0.0.tgz", diff --git a/package.json b/package.json index 90d7bfd..05bdb79 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "dependencies": { "@breezystack/lamejs": "^1.2.7", "class-transformer": "^0.5.1", + "fft.js": "^4.0.4", "idb": "^8.0.3", "jszip": "^3.10.1", "openai": "^6.33.0", diff --git a/src/components/MainContent.tsx b/src/components/MainContent.tsx index d5cce65..444155a 100644 --- a/src/components/MainContent.tsx +++ b/src/components/MainContent.tsx @@ -41,9 +41,12 @@ const MainContent: React.FC = ({ activeRegionId, setShowPianoRoll, setActiveRegionId, + pianoRollMode, + openSpectrogramViewer, addTrack, addAudioTrack, projectName, + savedProjectName, } = useProjectStore(); // State to store regions @@ -533,6 +536,12 @@ const MainContent: React.FC = ({ setShowPianoRoll(true); }; + // Handle spectrogram viewer open + const handleOpenSpectrogram = (regionId: string) => { + handleRegionClick(regionId); + openSpectrogramViewer(regionId); + }; + // Handle piano roll close const handlePianoRollClose = () => { setShowPianoRoll(false); @@ -824,16 +833,39 @@ const MainContent: React.FC = ({ onRegionUpdated={handleRegionUpdated} onRegionClick={handleRegionClick} onOpenPianoRoll={handleOpenPianoRoll} + onOpenSpectrogram={handleOpenSpectrogram} onExternalDropComplete={handleExternalDropComplete} /> - {/* Piano Roll - render using portal */} + {/* Piano Roll / Spectrogram Viewer - render using portal */} {showPianoRoll && createPortal( { + for (const track of tracks) { + const region = track.getRegions().find(r => r.getId() === activeRegionId); + if (region && region.getCurrentType() === 'KGAudioRegion') { + return region as unknown as KGAudioRegion; + } + } + return undefined; + })() + : undefined} + trackId={pianoRollMode === 'spectrogram' && activeRegionId + ? (() => { + for (const track of tracks) { + const region = track.getRegions().find(r => r.getId() === activeRegionId); + if (region) return track.getId().toString(); + } + return undefined; + })() + : undefined} + projectName={savedProjectName} />, document.body )} diff --git a/src/components/piano-roll/PianoGrid.tsx b/src/components/piano-roll/PianoGrid.tsx index 14e833c..b21d2c3 100644 --- a/src/components/piano-roll/PianoGrid.tsx +++ b/src/components/piano-roll/PianoGrid.tsx @@ -6,6 +6,8 @@ import { isModifierKeyPressed } from '../../util/osUtil'; import { generatePianoGridBackground, getMatchingChordsForPitch } from '../../util/scaleUtil'; import type { KeySignature } from '../../core/KGProject'; import { KGPianoRollState } from '../../core/state/KGPianoRollState'; +import SpectrogramCanvas from './SpectrogramCanvas'; +import type { KGAudioRegion } from '../../core/region/KGAudioRegion'; interface PianoGridProps { gridRef: MutableRefObject; @@ -24,6 +26,12 @@ interface PianoGridProps { selectedMode: string; keySignature: KeySignature; chordGuide: string; + audioRegion?: KGAudioRegion; + trackId?: string; + projectName?: string; + bpm?: number; + spectrogramThresholdDb?: number; + spectrogramPower?: number; } interface CursorPosition { @@ -44,7 +52,13 @@ const PianoGrid: React.FC = ({ regionStartBeat = 0, selectedMode, keySignature, - chordGuide + chordGuide, + audioRegion, + trackId, + projectName, + bpm = 120, + spectrogramThresholdDb = -25, + spectrogramPower = 0.5, }) => { const [cursorPosition, setCursorPosition] = useState(null); const [isModifierPressed, setIsModifierPressed] = useState(false); @@ -209,13 +223,25 @@ const PianoGrid: React.FC = ({
onMouseDown(e)} onMouseMove={handleMouseMove} onMouseLeave={handleMouseLeave} > + {/* Spectrogram layer — rendered at z-index 0, behind all highlights and notes */} + {audioRegion && trackId && projectName && ( + + )} + {/* Cursor Highlights */} {cursorPosition && ( <> diff --git a/src/components/piano-roll/PianoRoll.css b/src/components/piano-roll/PianoRoll.css index f6d9536..d4929b7 100644 --- a/src/components/piano-roll/PianoRoll.css +++ b/src/components/piano-roll/PianoRoll.css @@ -96,6 +96,36 @@ left: 0; } +/* Spectrogram toolbar controls */ +.spectrogram-toolbar-controls { + display: flex; + align-items: center; + gap: 6px; +} + +.spectrogram-control-label { + font-size: 10px; + color: #aaa; + white-space: nowrap; +} + +.spectrogram-threshold-slider { + /* Match the visual width of the Qua. Pos. / Qua. Len. buttons (~80px) */ + width: 80px; + height: 4px; + accent-color: #7a9ccf; + cursor: pointer; + margin: 0; +} + +.spectrogram-threshold-value { + font-size: 10px; + color: #e0e0e0; + min-width: 44px; + text-align: right; + white-space: nowrap; +} + .piano-roll-title { flex: 1; text-align: center; diff --git a/src/components/piano-roll/PianoRoll.tsx b/src/components/piano-roll/PianoRoll.tsx index 2fe27a5..990aed1 100644 --- a/src/components/piano-roll/PianoRoll.tsx +++ b/src/components/piano-roll/PianoRoll.tsx @@ -4,6 +4,7 @@ import type { MouseEvent } from 'react'; import { useProjectStore } from '../../stores/projectStore'; import { FaGripLines } from 'react-icons/fa'; import { KGMidiRegion } from '../../core/region/KGMidiRegion'; +import type { KGAudioRegion } from '../../core/region/KGAudioRegion'; import { DEBUG_MODE, PIANO_ROLL_CONSTANTS } from '../../constants'; import PianoRollHeader from './PianoRollHeader'; import PianoRollToolbar from './PianoRollToolbar'; @@ -22,18 +23,31 @@ interface PianoRollProps { regionId: string | null; initialPosition?: { x: number; y: number }; initialSize?: { width: number; height: number }; + mode?: 'midi-edit' | 'spectrogram'; + audioRegion?: KGAudioRegion; + trackId?: string; + projectName?: string; } const PianoRoll: React.FC = ({ onClose, regionId, initialPosition, - initialSize + initialSize, + mode = 'midi-edit', + audioRegion, + trackId, + projectName, }) => { - const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled } = useProjectStore(); + const isSpectrogram = mode === 'spectrogram'; + const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled, bpm } = useProjectStore(); // Tool state for piano roll const [activeTool, setActiveTool] = useState<'pointer' | 'pencil'>('pointer'); + + // Spectrogram controls (only used in spectrogram mode) + const [spectrogramThresholdDb, setSpectrogramThresholdDb] = useState(-25); + const [spectrogramPower, setSpectrogramPower] = useState(0.5); // Quantization state const [quantPosition, setQuantPosition] = useState('1/8'); @@ -855,6 +869,7 @@ const PianoRoll: React.FC = ({ // Get the title for the piano roll based on the active region const getPianoRollTitle = () => { + if (isSpectrogram) return audioRegion ? `SPECTROGRAM — ${audioRegion.getName()}` : 'SPECTROGRAM'; if (!activeRegion) return "EDIT NOTE CLIP"; // Calculate the bar and beat position of the region @@ -901,8 +916,13 @@ const PianoRoll: React.FC = ({ chordGuide={chordGuide} onChordGuideChange={handleChordGuideSelect} blinkButton={blinkButton} + mode={mode} + thresholdDb={spectrogramThresholdDb} + onThresholdChange={setSpectrogramThresholdDb} + power={spectrogramPower} + onPowerChange={setSpectrogramPower} /> - + = ({ selectedMode={selectedMode} keySignature={keySignature} chordGuide={chordGuide} + mode={mode} + audioRegion={audioRegion} + trackId={trackId} + projectName={projectName} + bpm={bpm} + spectrogramThresholdDb={spectrogramThresholdDb} + spectrogramPower={spectrogramPower} />
; @@ -26,6 +27,13 @@ interface PianoRollContentProps { selectedMode: string; keySignature: KeySignature; chordGuide: string; + mode?: 'midi-edit' | 'spectrogram'; + audioRegion?: KGAudioRegion; + trackId?: string; + projectName?: string; + bpm?: number; + spectrogramThresholdDb?: number; + spectrogramPower?: number; } const PianoRollContent: React.FC = ({ @@ -40,8 +48,16 @@ const PianoRollContent: React.FC = ({ onSetDeleteNotesTrigger, selectedMode, keySignature, - chordGuide + chordGuide, + mode = 'midi-edit', + audioRegion, + trackId, + projectName, + bpm = 120, + spectrogramThresholdDb = -25, + spectrogramPower = 0.5, }) => { + const isSpectrogram = mode === 'spectrogram'; // Get KGCore instance const core = KGCore.instance(); @@ -105,9 +121,8 @@ const PianoRollContent: React.FC = ({ // Combined click handler for both pointer and pencil modes const handleCombinedClick = (e: React.MouseEvent) => { - // Handle selection click (pointer mode) + if (isSpectrogram) return; handleBackgroundClick(e); - // Handle pencil mode note creation handleGridClick(e); }; @@ -144,7 +159,7 @@ const PianoRollContent: React.FC = ({ // Memoize the notes rendering to prevent unnecessary recalculations const memoizedNotes = useMemo(() => { - if (!activeRegion) return null; + if (isSpectrogram || !activeRegion) return null; if (DEBUG_MODE.PIANO_ROLL) { console.log(`Rendering notes for region: ${activeRegion.getId()}`); @@ -251,18 +266,24 @@ const PianoRollContent: React.FC = ({ {} : handleGridDoubleClick} + onClick={isSpectrogram ? () => {} : handleCombinedClick} + onMouseDown={isSpectrogram ? () => {} : handleBackgroundMouseDown} + isBoxSelecting={isSpectrogram ? false : isBoxSelectingRef.current} + selectionBox={isSpectrogram ? { startX: 0, startY: 0, endX: 0, endY: 0 } : selectionBoxRef.current} regionStartBeat={activeRegion?.getStartFromBeat() || 0} selectedMode={selectedMode} keySignature={keySignature} chordGuide={chordGuide} + audioRegion={audioRegion} + trackId={trackId} + projectName={projectName} + bpm={bpm} + spectrogramThresholdDb={spectrogramThresholdDb} + spectrogramPower={spectrogramPower} > {memoizedNotes} - {recordingNoteOverlays} + {!isSpectrogram && recordingNoteOverlays}
diff --git a/src/components/piano-roll/PianoRollToolbar.tsx b/src/components/piano-roll/PianoRollToolbar.tsx index ed12ea0..4df21b6 100644 --- a/src/components/piano-roll/PianoRollToolbar.tsx +++ b/src/components/piano-roll/PianoRollToolbar.tsx @@ -4,6 +4,13 @@ import { KGDropdown } from '../common'; import { KGPianoRollState } from '../../core/state/KGPianoRollState'; import { KGCore } from '../../core/KGCore'; +const POWER_OPTIONS = [ + { label: 'Linear', value: '1.0' }, + { label: '√ (default)', value: '0.5' }, + { label: 'Mild', value: '0.4' }, + { label: 'Strong', value: '0.3' }, +]; + interface PianoRollToolbarProps { activeTool: 'pointer' | 'pencil'; onToolSelect: (tool: 'pointer' | 'pencil') => void; @@ -17,6 +24,11 @@ interface PianoRollToolbarProps { chordGuide: string; onChordGuideChange: (value: string) => void; blinkButton?: string | null; + mode?: 'midi-edit' | 'spectrogram'; + thresholdDb?: number; + onThresholdChange?: (db: number) => void; + power?: number; + onPowerChange?: (power: number) => void; } const PianoRollToolbar: React.FC = ({ @@ -31,82 +43,117 @@ const PianoRollToolbar: React.FC = ({ onModeChange, chordGuide, onChordGuideChange, - blinkButton = null + blinkButton = null, + mode = 'midi-edit', + thresholdDb = -25, + onThresholdChange, + power = 0.5, + onPowerChange, }) => { + const isSpectrogram = mode === 'spectrogram'; + return (
-
- {/* Left section with mode and chord guide dropdowns */} - ({ label: data.name, value: id }))} - value={selectedMode} - onChange={(value) => onModeChange(value)} - label="Mode" - buttonClassName="mode-dropdown" - showValueAsLabel={true} - /> - onChordGuideChange(value)} - label="Chord" - buttonClassName="chord-guide-dropdown" - showValueAsLabel={true} - /> -
- -
- {/* Center section with pointer and pencil tools */} - - -
- + {!isSpectrogram && ( +
+ ({ label: data.name, value: id }))} + value={selectedMode} + onChange={(value) => onModeChange(value)} + label="Mode" + buttonClassName="mode-dropdown" + showValueAsLabel={true} + /> + onChordGuideChange(value)} + label="Chord" + buttonClassName="chord-guide-dropdown" + showValueAsLabel={true} + /> +
+ )} + + {!isSpectrogram && ( +
+ + +
+ )} +
- {/* Right section with quantization options */} - onSnappingSelect(value)} - label="Snap" - buttonClassName="snapping" - showValueAsLabel={true} - /> + {!isSpectrogram && ( + <> + onSnappingSelect(value)} + label="Snap" + buttonClassName="snapping" + showValueAsLabel={true} + /> + onQuantSelect('position', value)} + label="Qua. Pos." + buttonClassName={`quant-position ${blinkButton === 'quant-position' ? 'button-blink' : ''}`} + /> + onQuantSelect('length', value)} + label="Qua. Len." + buttonClassName={`quant-length ${blinkButton === 'quant-length' ? 'button-blink' : ''}`} + /> + + )} - onQuantSelect('position', value)} - label="Qua. Pos." - buttonClassName={`quant-position ${blinkButton === 'quant-position' ? 'button-blink' : ''}`} - /> - - onQuantSelect('length', value)} - label="Qua. Len." - buttonClassName={`quant-length ${blinkButton === 'quant-length' ? 'button-blink' : ''}`} - /> + {isSpectrogram && ( +
+ Floor + onThresholdChange?.(parseInt(e.target.value))} + title={`Noise floor: ${thresholdDb} dB`} + /> + {thresholdDb} dB + onPowerChange?.(parseFloat(v))} + label="Curve" + buttonClassName="curve-dropdown" + showValueAsLabel={true} + /> +
+ )}
); }; -export default PianoRollToolbar; \ No newline at end of file +export default PianoRollToolbar; diff --git a/src/components/piano-roll/SpectrogramCanvas.tsx b/src/components/piano-roll/SpectrogramCanvas.tsx new file mode 100644 index 0000000..6b4b90f --- /dev/null +++ b/src/components/piano-roll/SpectrogramCanvas.tsx @@ -0,0 +1,256 @@ +import React, { useCallback, useEffect, useRef, useState } from 'react'; +import * as Tone from 'tone'; +import { KGAudioRegion } from '../../core/region/KGAudioRegion'; +import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface'; +import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage'; +import type { SpectrogramRequest, SpectrogramResult } from '../../workers/spectrogramWorker'; + +interface SpectrogramCanvasProps { + audioRegion: KGAudioRegion; + trackId: string; + projectName: string; + bpm: number; + thresholdDb: number; + power: number; +} + +const PITCH_BINS = 128; +const HOP_SIZE = 1024; + +// Piecewise-linear RGB colormap: black → dark blue → blue → purple → red → orange → yellow +// Hue rotates 240°→300°→0°→60°, bypassing green entirely. +const COLORMAP_STOPS: Array<[number, [number, number, number]]> = [ + [0.00, [ 0, 0, 0]], + [0.15, [ 0, 0, 180]], + [0.35, [ 0, 60, 255]], + [0.55, [180, 0, 120]], + [0.70, [255, 0, 0]], + [0.85, [255, 140, 0]], + [1.00, [255, 255, 0]], +]; + +function hotColormap(v: number): [number, number, number] { + v = Math.max(0, Math.min(1, v)); + for (let i = 0; i < COLORMAP_STOPS.length - 1; i++) { + const [t0, c0] = COLORMAP_STOPS[i]; + const [t1, c1] = COLORMAP_STOPS[i + 1]; + if (v <= t1) { + const t = (v - t0) / (t1 - t0); + return [ + Math.round(c0[0] + t * (c1[0] - c0[0])), + Math.round(c0[1] + t * (c1[1] - c0[1])), + Math.round(c0[2] + t * (c1[2] - c0[2])), + ]; + } + } + return COLORMAP_STOPS[COLORMAP_STOPS.length - 1][1]; +} + +const SpectrogramCanvas: React.FC = ({ + audioRegion, + trackId, + projectName, + bpm, + thresholdDb, + power, +}) => { + const canvasRef = useRef(null); + const [loading, setLoading] = useState(true); + const workerRef = useRef(null); + + // Cache the raw linear result so threshold/power changes re-render without re-running FFT + const rawResultRef = useRef(null); + const sampleRateRef = useRef(44100); + const regionDurationRef = useRef(0); + + const renderSpectrogram = useCallback(( + result: SpectrogramResult, + sampleRate: number, + regionDurationSeconds: number, + thresholdDb: number, + power: number, + ) => { + const canvas = canvasRef.current; + if (!canvas) return; + const ctx = canvas.getContext('2d'); + if (!ctx) return; + + const beatWidth = + parseInt(getComputedStyle(document.documentElement).getPropertyValue('--region-grid-beat-width')) || 40; + const noteHeight = + parseInt(getComputedStyle(document.documentElement).getPropertyValue('--region-piano-key-height')) || 20; + + const totalBeats = (regionDurationSeconds * bpm) / 60; + const canvasWidth = Math.ceil(totalBeats * beatWidth); + const canvasHeight = PITCH_BINS * noteHeight; + + // Convert dB threshold to linear: values below this → black + const linearThreshold = Math.pow(10, thresholdDb / 20); + + // 1. Paint at natural STFT resolution onto an offscreen canvas (timeSteps × 128). + // Row i = pitchIndex i = pitch (107 − i). Apply threshold then power curve. + const offscreen = document.createElement('canvas'); + offscreen.width = result.timeSteps; + offscreen.height = PITCH_BINS; + const offCtx = offscreen.getContext('2d'); + if (!offCtx) return; + + const imgData = offCtx.createImageData(result.timeSteps, PITCH_BINS); + const pixels = imgData.data; + + for (let row = 0; row < PITCH_BINS; row++) { + const pitch = 107 - row; + for (let col = 0; col < result.timeSteps; col++) { + const idx = (row * result.timeSteps + col) * 4; + pixels[idx + 3] = 255; // always opaque + + if (pitch < 12 || pitch > 107) continue; // outside range → black + + const raw = result.data[col * PITCH_BINS + pitch]; + + // Hard threshold: values below noise floor → 0 (black) + // Re-scale surviving range to [0,1] then apply power curve + const gated = raw < linearThreshold + ? 0 + : Math.pow((raw - linearThreshold) / (1 - linearThreshold), power); + + if (gated <= 0) continue; // stays black + + const [r, g, b] = hotColormap(gated); + pixels[idx] = r; + pixels[idx + 1] = g; + pixels[idx + 2] = b; + } + } + + offCtx.putImageData(imgData, 0, 0); + + // 2. Stretch onto the full canvas — browser bilinear filter smooths between bins. + canvas.width = canvasWidth; + canvas.height = canvasHeight; + ctx.imageSmoothingEnabled = true; + ctx.imageSmoothingQuality = 'high'; + ctx.drawImage(offscreen, 0, 0, canvasWidth, canvasHeight); + }, [bpm]); + + // Re-render without re-running the worker when threshold or power changes + useEffect(() => { + if (rawResultRef.current) { + renderSpectrogram( + rawResultRef.current, + sampleRateRef.current, + regionDurationRef.current, + thresholdDb, + power, + ); + } + }, [thresholdDb, power, renderSpectrogram]); + + // Load audio + run worker when the audio region itself changes + useEffect(() => { + let cancelled = false; + + const compute = async () => { + setLoading(true); + workerRef.current?.terminate(); + workerRef.current = null; + + try { + let audioBuffer: AudioBuffer | undefined = KGAudioInterface.instance().getAudioBuffer( + trackId, + audioRegion.getAudioFileId() + ); + + if (!audioBuffer) { + const arrayBuffer = await KGAudioFileStorage.loadAudioFile( + projectName, + audioRegion.getAudioFileId() + ); + if (cancelled) return; + const actx = Tone.getContext().rawContext as AudioContext; + audioBuffer = await actx.decodeAudioData(arrayBuffer); + } + + if (cancelled) return; + + const pcm = audioBuffer.getChannelData(0); + const sampleRate = audioBuffer.sampleRate; + const regionDurationSeconds = (audioRegion.getLength() * 60) / bpm; + + const worker = new Worker( + new URL('../../workers/spectrogramWorker.ts', import.meta.url), + { type: 'module' } + ); + workerRef.current = worker; + + worker.onmessage = (e: MessageEvent) => { + if (cancelled) { worker.terminate(); return; } + rawResultRef.current = e.data; + sampleRateRef.current = sampleRate; + regionDurationRef.current = regionDurationSeconds; + renderSpectrogram(e.data, sampleRate, regionDurationSeconds, thresholdDb, power); + setLoading(false); + worker.terminate(); + workerRef.current = null; + }; + + const request: SpectrogramRequest = { + pcm: pcm.slice(0) as Float32Array, + sampleRate, + clipStartOffsetSeconds: audioRegion.getClipStartOffsetSeconds(), + regionDurationSeconds, + bpm, + }; + worker.postMessage(request, [request.pcm.buffer]); + } catch (err) { + if (!cancelled) { + console.error('SpectrogramCanvas: failed to compute spectrogram', err); + setLoading(false); + } + } + }; + + compute(); + + return () => { + cancelled = true; + workerRef.current?.terminate(); + workerRef.current = null; + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [audioRegion, trackId, projectName, bpm]); + + return ( + <> + + {loading && ( +
+ Computing spectrogram… +
+ )} + + ); +}; + +export default SpectrogramCanvas; diff --git a/src/components/track/Region.css b/src/components/track/Region.css index 9175075..ff97412 100644 --- a/src/components/track/Region.css +++ b/src/components/track/Region.css @@ -119,6 +119,27 @@ background: rgba(0, 0, 0, 0.35); } +.region-spectrogram-btn { + position: absolute; + top: 4px; + left: 4px; + background: rgba(0, 0, 0, 0.25); + color: #fff; + border: 1px solid rgba(255, 255, 255, 0.2); + border-radius: 3px; + padding: 2px; + margin: 0; + cursor: pointer; + z-index: 2; + display: inline-flex; + align-items: center; + justify-content: center; +} + +.region-spectrogram-btn:hover { + background: rgba(0, 0, 0, 0.35); +} + /* Instrument dropdown specific styles */ .instrument-dropdown .quant-dropdown { min-width: 80px; diff --git a/src/components/track/RegionItem.tsx b/src/components/track/RegionItem.tsx index d6f6934..5a4f0b9 100644 --- a/src/components/track/RegionItem.tsx +++ b/src/components/track/RegionItem.tsx @@ -1,6 +1,7 @@ import React, { useState, useRef, useEffect } from 'react'; import './Region.css'; import { FaPencilAlt } from 'react-icons/fa'; +import { MdGraphicEq } from 'react-icons/md'; import type { ResizeAction } from '../interfaces'; import { REGION_CONSTANTS, DEBUG_MODE } from '../../constants'; import { KGMidiRegion } from '../../core/region/KGMidiRegion'; @@ -27,6 +28,8 @@ interface RegionItemProps { onClick?: (regionId: string) => void; // Explicit open piano roll action from header pencil icon onOpenPianoRoll?: (regionId: string) => void; + // Open spectrogram viewer for audio regions + onOpenSpectrogram?: (regionId: string) => void; // MIDI region data for rendering notes midiRegion?: KGMidiRegion; // Audio region data for rendering waveform @@ -49,6 +52,7 @@ const RegionItem: React.FC = ({ onDragEnd, onClick, onOpenPianoRoll, + onOpenSpectrogram, midiRegion, audioRegion, audioBuffer @@ -576,6 +580,26 @@ const RegionItem: React.FC = ({ )} + {audioRegion && ( + + )} diff --git a/src/components/track/TrackGridItem.tsx b/src/components/track/TrackGridItem.tsx index 08ee19b..8f322d3 100644 --- a/src/components/track/TrackGridItem.tsx +++ b/src/components/track/TrackGridItem.tsx @@ -26,6 +26,7 @@ interface TrackGridItemProps { onRegionDragEnd?: (regionId: string, finalBarNumber: number, finalTrackIndex: number) => void; onRegionClick?: (regionId: string) => void; onOpenPianoRoll?: (regionId: string) => void; + onOpenSpectrogram?: (regionId: string) => void; allTracks?: KGTrack[]; // Added to access all tracks for drag operations onKGOneClipDrop?: (e: React.DragEvent, trackIndex: number) => void; } @@ -47,6 +48,7 @@ const TrackGridItem: React.FC = ({ onRegionDragEnd, onRegionClick, onOpenPianoRoll, + onOpenSpectrogram, allTracks, onKGOneClipDrop, }) => { @@ -566,6 +568,9 @@ const TrackGridItem: React.FC = ({ onRegionClick(regionId); } }} + onOpenSpectrogram={audioRegion ? (regionId) => { + onOpenSpectrogram?.(regionId); + } : undefined} midiRegion={midiRegion} audioRegion={audioRegion} audioBuffer={audioBuffer} diff --git a/src/components/track/TrackGridPanel.tsx b/src/components/track/TrackGridPanel.tsx index da64b29..875895e 100644 --- a/src/components/track/TrackGridPanel.tsx +++ b/src/components/track/TrackGridPanel.tsx @@ -30,6 +30,7 @@ interface TrackGridPanelProps { onRegionUpdated?: (regionId: string, updates: Partial, expectedModelUpdates?: { startBeat: number, length: number }) => void; onRegionClick?: (regionId: string) => void; onOpenPianoRoll?: (regionId: string) => void; + onOpenSpectrogram?: (regionId: string) => void; onExternalDropComplete?: (trackIndex: number, regionUI: RegionUI) => void; } @@ -46,6 +47,7 @@ const TrackGridPanel: React.FC = ({ onRegionUpdated, onRegionClick, onOpenPianoRoll, + onOpenSpectrogram, onExternalDropComplete, }) => { const gridContainerRef = useRef(null); @@ -651,6 +653,7 @@ const TrackGridPanel: React.FC = ({ onRegionDragEnd={handleRegionDragEnd} onRegionClick={handleRegionClick} onOpenPianoRoll={onOpenPianoRoll} + onOpenSpectrogram={onOpenSpectrogram} allTracks={tracks} onKGOneClipDrop={handleExternalDrop} /> diff --git a/src/stores/projectStore.ts b/src/stores/projectStore.ts index 9d1f1a1..0e2f50d 100644 --- a/src/stores/projectStore.ts +++ b/src/stores/projectStore.ts @@ -78,6 +78,7 @@ interface ProjectState { // Piano roll state showPianoRoll: boolean; activeRegionId: string | null; + pianoRollMode: 'midi-edit' | 'spectrogram'; // ChatBox state showChatBox: boolean; @@ -146,6 +147,8 @@ interface ProjectState { // Piano roll actions setShowPianoRoll: (show: boolean) => void; setActiveRegionId: (regionId: string | null) => void; + openMidiPianoRoll: (regionId: string) => void; + openSpectrogramViewer: (regionId: string) => void; // Project state cleanup cleanupProjectState: () => void; @@ -304,6 +307,7 @@ export const useProjectStore = create((set, get) => { // Initial piano roll state showPianoRoll: false, activeRegionId: null, + pianoRollMode: 'midi-edit' as const, // Initial ChatBox state showChatBox: initialChatBoxState, @@ -1028,10 +1032,18 @@ export const useProjectStore = create((set, get) => { setShowPianoRoll: (show: boolean) => { set({ showPianoRoll: show }); }, - + setActiveRegionId: (regionId: string | null) => { set({ activeRegionId: regionId }); }, + + openMidiPianoRoll: (regionId: string) => { + set({ showPianoRoll: true, activeRegionId: regionId, pianoRollMode: 'midi-edit' }); + }, + + openSpectrogramViewer: (regionId: string) => { + set({ showPianoRoll: true, activeRegionId: regionId, pianoRollMode: 'spectrogram' }); + }, // Project state cleanup - used when starting new/loading projects cleanupProjectState: () => { diff --git a/src/workers/spectrogramWorker.ts b/src/workers/spectrogramWorker.ts new file mode 100644 index 0000000..eba58f4 --- /dev/null +++ b/src/workers/spectrogramWorker.ts @@ -0,0 +1,96 @@ +import FFT from 'fft.js'; + +export interface SpectrogramRequest { + pcm: Float32Array; + sampleRate: number; + clipStartOffsetSeconds: number; + regionDurationSeconds: number; + bpm: number; +} + +export interface SpectrogramResult { + data: Float32Array; // [timeSteps × 128] row-major, row = time step, col = pitch 0-127 + timeSteps: number; +} + +const FFT_SIZE = 8192; +const HOP_SIZE = 1024; +const PITCH_BINS = 128; + +function hannWindow(size: number): Float32Array { + const w = new Float32Array(size); + for (let i = 0; i < size; i++) { + w[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / (size - 1))); + } + return w; +} + +self.onmessage = (e: MessageEvent) => { + const { pcm, sampleRate, clipStartOffsetSeconds, regionDurationSeconds } = e.data; + + const startSample = Math.floor(clipStartOffsetSeconds * sampleRate); + const endSample = Math.min(pcm.length, startSample + Math.ceil(regionDurationSeconds * sampleRate)); + const regionSamples = pcm.subarray(startSample, endSample); + + const fft = new FFT(FFT_SIZE); + const hann = hannWindow(FFT_SIZE); + const complexOut = fft.createComplexArray() as number[]; + const inputPadded = new Float32Array(FFT_SIZE); + + const totalHops = Math.max(1, Math.ceil((regionSamples.length - FFT_SIZE) / HOP_SIZE) + 1); + const result = new Float32Array(totalHops * PITCH_BINS); + + let maxVal = 0; + + for (let hop = 0; hop < totalHops; hop++) { + const offset = hop * HOP_SIZE; + + // Fill windowed frame (zero-pad at end if needed) + inputPadded.fill(0); + const available = Math.min(FFT_SIZE, regionSamples.length - offset); + for (let i = 0; i < available; i++) { + inputPadded[i] = regionSamples[offset + i] * hann[i]; + } + + fft.realTransform(complexOut, inputPadded as unknown as number[]); + fft.completeSpectrum(complexOut); + + // Max-pool magnitude into pitch bins: keep the loudest FFT bin per semitone, + // rather than summing. Summing inflates every bin by how many FFT bins land there. + const pitchRow = hop * PITCH_BINS; + const numBins = FFT_SIZE / 2; + + for (let bin = 1; bin < numBins; bin++) { + const freq = (bin * sampleRate) / FFT_SIZE; + if (freq < 20 || freq > 20000) continue; + + // Convert frequency to MIDI pitch; clamp to piano roll range C0–B7 (MIDI 12–107) + const pitch = Math.round(69 + 12 * Math.log2(freq / 440)); + if (pitch < 12 || pitch > 107) continue; + + const re = complexOut[2 * bin]; + const im = complexOut[2 * bin + 1]; + const magnitude = Math.sqrt(re * re + im * im); + + if (magnitude > result[pitchRow + pitch]) { + result[pitchRow + pitch] = magnitude; + } + } + + // Track global max for normalization + for (let p = 0; p < PITCH_BINS; p++) { + if (result[pitchRow + p] > maxVal) maxVal = result[pitchRow + p]; + } + } + + // Linear normalization only — threshold and power curve are applied in the canvas + // renderer so changing them is instant (no need to re-run the FFT). + if (maxVal > 0) { + for (let i = 0; i < result.length; i++) { + result[i] = result[i] / maxVal; + } + } + + const response: SpectrogramResult = { data: result, timeSteps: totalHops }; + self.postMessage(response, [result.buffer]); +};