feat: added audio track spectrogram visualization feature
This commit is contained in:
Xiaohan-Tian
Generated
+9
-2
@@ -1,15 +1,16 @@
|
||||
{
|
||||
"name": "K.G.Studio",
|
||||
"version": "0.10.0-build.20260411",
|
||||
"version": "0.12.0-build.20260430",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "K.G.Studio",
|
||||
"version": "0.10.0-build.20260411",
|
||||
"version": "0.12.0-build.20260430",
|
||||
"dependencies": {
|
||||
"@breezystack/lamejs": "^1.2.7",
|
||||
"class-transformer": "^0.5.1",
|
||||
"fft.js": "^4.0.4",
|
||||
"idb": "^8.0.3",
|
||||
"jszip": "^3.10.1",
|
||||
"openai": "^6.33.0",
|
||||
@@ -4691,6 +4692,12 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fft.js": {
|
||||
"version": "4.0.4",
|
||||
"resolved": "https://registry.npmjs.org/fft.js/-/fft.js-4.0.4.tgz",
|
||||
"integrity": "sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/figures": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/figures/-/figures-5.0.0.tgz",
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
"dependencies": {
|
||||
"@breezystack/lamejs": "^1.2.7",
|
||||
"class-transformer": "^0.5.1",
|
||||
"fft.js": "^4.0.4",
|
||||
"idb": "^8.0.3",
|
||||
"jszip": "^3.10.1",
|
||||
"openai": "^6.33.0",
|
||||
|
||||
@@ -41,9 +41,12 @@ const MainContent: React.FC<MainContentProps> = ({
|
||||
activeRegionId,
|
||||
setShowPianoRoll,
|
||||
setActiveRegionId,
|
||||
pianoRollMode,
|
||||
openSpectrogramViewer,
|
||||
addTrack,
|
||||
addAudioTrack,
|
||||
projectName,
|
||||
savedProjectName,
|
||||
} = useProjectStore();
|
||||
|
||||
// State to store regions
|
||||
@@ -533,6 +536,12 @@ const MainContent: React.FC<MainContentProps> = ({
|
||||
setShowPianoRoll(true);
|
||||
};
|
||||
|
||||
// Handle spectrogram viewer open
|
||||
const handleOpenSpectrogram = (regionId: string) => {
|
||||
handleRegionClick(regionId);
|
||||
openSpectrogramViewer(regionId);
|
||||
};
|
||||
|
||||
// Handle piano roll close
|
||||
const handlePianoRollClose = () => {
|
||||
setShowPianoRoll(false);
|
||||
@@ -824,16 +833,39 @@ const MainContent: React.FC<MainContentProps> = ({
|
||||
onRegionUpdated={handleRegionUpdated}
|
||||
onRegionClick={handleRegionClick}
|
||||
onOpenPianoRoll={handleOpenPianoRoll}
|
||||
onOpenSpectrogram={handleOpenSpectrogram}
|
||||
onExternalDropComplete={handleExternalDropComplete}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Piano Roll - render using portal */}
|
||||
{/* Piano Roll / Spectrogram Viewer - render using portal */}
|
||||
{showPianoRoll && createPortal(
|
||||
<PianoRoll
|
||||
onClose={handlePianoRollClose}
|
||||
regionId={activeRegionId}
|
||||
mode={pianoRollMode}
|
||||
audioRegion={pianoRollMode === 'spectrogram' && activeRegionId
|
||||
? (() => {
|
||||
for (const track of tracks) {
|
||||
const region = track.getRegions().find(r => r.getId() === activeRegionId);
|
||||
if (region && region.getCurrentType() === 'KGAudioRegion') {
|
||||
return region as unknown as KGAudioRegion;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
})()
|
||||
: undefined}
|
||||
trackId={pianoRollMode === 'spectrogram' && activeRegionId
|
||||
? (() => {
|
||||
for (const track of tracks) {
|
||||
const region = track.getRegions().find(r => r.getId() === activeRegionId);
|
||||
if (region) return track.getId().toString();
|
||||
}
|
||||
return undefined;
|
||||
})()
|
||||
: undefined}
|
||||
projectName={savedProjectName}
|
||||
/>,
|
||||
document.body
|
||||
)}
|
||||
|
||||
@@ -6,6 +6,8 @@ import { isModifierKeyPressed } from '../../util/osUtil';
|
||||
import { generatePianoGridBackground, getMatchingChordsForPitch } from '../../util/scaleUtil';
|
||||
import type { KeySignature } from '../../core/KGProject';
|
||||
import { KGPianoRollState } from '../../core/state/KGPianoRollState';
|
||||
import SpectrogramCanvas from './SpectrogramCanvas';
|
||||
import type { KGAudioRegion } from '../../core/region/KGAudioRegion';
|
||||
|
||||
interface PianoGridProps {
|
||||
gridRef: MutableRefObject<HTMLDivElement | null>;
|
||||
@@ -24,6 +26,12 @@ interface PianoGridProps {
|
||||
selectedMode: string;
|
||||
keySignature: KeySignature;
|
||||
chordGuide: string;
|
||||
audioRegion?: KGAudioRegion;
|
||||
trackId?: string;
|
||||
projectName?: string;
|
||||
bpm?: number;
|
||||
spectrogramThresholdDb?: number;
|
||||
spectrogramPower?: number;
|
||||
}
|
||||
|
||||
interface CursorPosition {
|
||||
@@ -44,7 +52,13 @@ const PianoGrid: React.FC<PianoGridProps> = ({
|
||||
regionStartBeat = 0,
|
||||
selectedMode,
|
||||
keySignature,
|
||||
chordGuide
|
||||
chordGuide,
|
||||
audioRegion,
|
||||
trackId,
|
||||
projectName,
|
||||
bpm = 120,
|
||||
spectrogramThresholdDb = -25,
|
||||
spectrogramPower = 0.5,
|
||||
}) => {
|
||||
const [cursorPosition, setCursorPosition] = useState<CursorPosition | null>(null);
|
||||
const [isModifierPressed, setIsModifierPressed] = useState(false);
|
||||
@@ -209,13 +223,25 @@ const PianoGrid: React.FC<PianoGridProps> = ({
|
||||
<div
|
||||
className={`piano-grid ${isModifierPressed ? 'pencil-cursor' : ''}`}
|
||||
ref={gridRef}
|
||||
style={{ backgroundImage }}
|
||||
style={{ backgroundImage: audioRegion ? undefined : backgroundImage }}
|
||||
onDoubleClick={onDoubleClick}
|
||||
onClick={onClick}
|
||||
onMouseDown={(e) => onMouseDown(e)}
|
||||
onMouseMove={handleMouseMove}
|
||||
onMouseLeave={handleMouseLeave}
|
||||
>
|
||||
{/* Spectrogram layer — rendered at z-index 0, behind all highlights and notes */}
|
||||
{audioRegion && trackId && projectName && (
|
||||
<SpectrogramCanvas
|
||||
audioRegion={audioRegion}
|
||||
trackId={trackId}
|
||||
projectName={projectName}
|
||||
bpm={bpm}
|
||||
thresholdDb={spectrogramThresholdDb}
|
||||
power={spectrogramPower}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Cursor Highlights */}
|
||||
{cursorPosition && (
|
||||
<>
|
||||
|
||||
@@ -96,6 +96,36 @@
|
||||
left: 0;
|
||||
}
|
||||
|
||||
/* Spectrogram toolbar controls */
|
||||
.spectrogram-toolbar-controls {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.spectrogram-control-label {
|
||||
font-size: 10px;
|
||||
color: #aaa;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.spectrogram-threshold-slider {
|
||||
/* Match the visual width of the Qua. Pos. / Qua. Len. buttons (~80px) */
|
||||
width: 80px;
|
||||
height: 4px;
|
||||
accent-color: #7a9ccf;
|
||||
cursor: pointer;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.spectrogram-threshold-value {
|
||||
font-size: 10px;
|
||||
color: #e0e0e0;
|
||||
min-width: 44px;
|
||||
text-align: right;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.piano-roll-title {
|
||||
flex: 1;
|
||||
text-align: center;
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { MouseEvent } from 'react';
|
||||
import { useProjectStore } from '../../stores/projectStore';
|
||||
import { FaGripLines } from 'react-icons/fa';
|
||||
import { KGMidiRegion } from '../../core/region/KGMidiRegion';
|
||||
import type { KGAudioRegion } from '../../core/region/KGAudioRegion';
|
||||
import { DEBUG_MODE, PIANO_ROLL_CONSTANTS } from '../../constants';
|
||||
import PianoRollHeader from './PianoRollHeader';
|
||||
import PianoRollToolbar from './PianoRollToolbar';
|
||||
@@ -22,19 +23,32 @@ interface PianoRollProps {
|
||||
regionId: string | null;
|
||||
initialPosition?: { x: number; y: number };
|
||||
initialSize?: { width: number; height: number };
|
||||
mode?: 'midi-edit' | 'spectrogram';
|
||||
audioRegion?: KGAudioRegion;
|
||||
trackId?: string;
|
||||
projectName?: string;
|
||||
}
|
||||
|
||||
const PianoRoll: React.FC<PianoRollProps> = ({
|
||||
onClose,
|
||||
regionId,
|
||||
initialPosition,
|
||||
initialSize
|
||||
initialSize,
|
||||
mode = 'midi-edit',
|
||||
audioRegion,
|
||||
trackId,
|
||||
projectName,
|
||||
}) => {
|
||||
const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled } = useProjectStore();
|
||||
const isSpectrogram = mode === 'spectrogram';
|
||||
const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled, bpm } = useProjectStore();
|
||||
|
||||
// Tool state for piano roll
|
||||
const [activeTool, setActiveTool] = useState<'pointer' | 'pencil'>('pointer');
|
||||
|
||||
// Spectrogram controls (only used in spectrogram mode)
|
||||
const [spectrogramThresholdDb, setSpectrogramThresholdDb] = useState<number>(-25);
|
||||
const [spectrogramPower, setSpectrogramPower] = useState<number>(0.5);
|
||||
|
||||
// Quantization state
|
||||
const [quantPosition, setQuantPosition] = useState<string>('1/8');
|
||||
const [quantLength, setQuantLength] = useState<string>('1/8');
|
||||
@@ -855,6 +869,7 @@ const PianoRoll: React.FC<PianoRollProps> = ({
|
||||
|
||||
// Get the title for the piano roll based on the active region
|
||||
const getPianoRollTitle = () => {
|
||||
if (isSpectrogram) return audioRegion ? `SPECTROGRAM — ${audioRegion.getName()}` : 'SPECTROGRAM';
|
||||
if (!activeRegion) return "EDIT NOTE CLIP";
|
||||
|
||||
// Calculate the bar and beat position of the region
|
||||
@@ -901,6 +916,11 @@ const PianoRoll: React.FC<PianoRollProps> = ({
|
||||
chordGuide={chordGuide}
|
||||
onChordGuideChange={handleChordGuideSelect}
|
||||
blinkButton={blinkButton}
|
||||
mode={mode}
|
||||
thresholdDb={spectrogramThresholdDb}
|
||||
onThresholdChange={setSpectrogramThresholdDb}
|
||||
power={spectrogramPower}
|
||||
onPowerChange={setSpectrogramPower}
|
||||
/>
|
||||
|
||||
<PianoRollContent
|
||||
@@ -916,6 +936,13 @@ const PianoRoll: React.FC<PianoRollProps> = ({
|
||||
selectedMode={selectedMode}
|
||||
keySignature={keySignature}
|
||||
chordGuide={chordGuide}
|
||||
mode={mode}
|
||||
audioRegion={audioRegion}
|
||||
trackId={trackId}
|
||||
projectName={projectName}
|
||||
bpm={bpm}
|
||||
spectrogramThresholdDb={spectrogramThresholdDb}
|
||||
spectrogramPower={spectrogramPower}
|
||||
/>
|
||||
|
||||
<div
|
||||
|
||||
@@ -12,6 +12,7 @@ import PianoGrid from './PianoGrid';
|
||||
import { useNoteOperations } from '../../hooks/useNoteOperations';
|
||||
import { useNoteSelection } from '../../hooks/useNoteSelection';
|
||||
import type { KeySignature } from '../../core/KGProject';
|
||||
import type { KGAudioRegion } from '../../core/region/KGAudioRegion';
|
||||
|
||||
interface PianoRollContentProps {
|
||||
contentRef: React.MutableRefObject<HTMLDivElement | null>;
|
||||
@@ -26,6 +27,13 @@ interface PianoRollContentProps {
|
||||
selectedMode: string;
|
||||
keySignature: KeySignature;
|
||||
chordGuide: string;
|
||||
mode?: 'midi-edit' | 'spectrogram';
|
||||
audioRegion?: KGAudioRegion;
|
||||
trackId?: string;
|
||||
projectName?: string;
|
||||
bpm?: number;
|
||||
spectrogramThresholdDb?: number;
|
||||
spectrogramPower?: number;
|
||||
}
|
||||
|
||||
const PianoRollContent: React.FC<PianoRollContentProps> = ({
|
||||
@@ -40,8 +48,16 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
|
||||
onSetDeleteNotesTrigger,
|
||||
selectedMode,
|
||||
keySignature,
|
||||
chordGuide
|
||||
chordGuide,
|
||||
mode = 'midi-edit',
|
||||
audioRegion,
|
||||
trackId,
|
||||
projectName,
|
||||
bpm = 120,
|
||||
spectrogramThresholdDb = -25,
|
||||
spectrogramPower = 0.5,
|
||||
}) => {
|
||||
const isSpectrogram = mode === 'spectrogram';
|
||||
// Get KGCore instance
|
||||
const core = KGCore.instance();
|
||||
|
||||
@@ -105,9 +121,8 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
|
||||
|
||||
// Combined click handler for both pointer and pencil modes
|
||||
const handleCombinedClick = (e: React.MouseEvent) => {
|
||||
// Handle selection click (pointer mode)
|
||||
if (isSpectrogram) return;
|
||||
handleBackgroundClick(e);
|
||||
// Handle pencil mode note creation
|
||||
handleGridClick(e);
|
||||
};
|
||||
|
||||
@@ -144,7 +159,7 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
|
||||
|
||||
// Memoize the notes rendering to prevent unnecessary recalculations
|
||||
const memoizedNotes = useMemo(() => {
|
||||
if (!activeRegion) return null;
|
||||
if (isSpectrogram || !activeRegion) return null;
|
||||
|
||||
if (DEBUG_MODE.PIANO_ROLL) {
|
||||
console.log(`Rendering notes for region: ${activeRegion.getId()}`);
|
||||
@@ -251,18 +266,24 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
|
||||
|
||||
<PianoGrid
|
||||
gridRef={pianoGridRef}
|
||||
onDoubleClick={handleGridDoubleClick}
|
||||
onClick={handleCombinedClick}
|
||||
onMouseDown={handleBackgroundMouseDown}
|
||||
isBoxSelecting={isBoxSelectingRef.current}
|
||||
selectionBox={selectionBoxRef.current}
|
||||
onDoubleClick={isSpectrogram ? () => {} : handleGridDoubleClick}
|
||||
onClick={isSpectrogram ? () => {} : handleCombinedClick}
|
||||
onMouseDown={isSpectrogram ? () => {} : handleBackgroundMouseDown}
|
||||
isBoxSelecting={isSpectrogram ? false : isBoxSelectingRef.current}
|
||||
selectionBox={isSpectrogram ? { startX: 0, startY: 0, endX: 0, endY: 0 } : selectionBoxRef.current}
|
||||
regionStartBeat={activeRegion?.getStartFromBeat() || 0}
|
||||
selectedMode={selectedMode}
|
||||
keySignature={keySignature}
|
||||
chordGuide={chordGuide}
|
||||
audioRegion={audioRegion}
|
||||
trackId={trackId}
|
||||
projectName={projectName}
|
||||
bpm={bpm}
|
||||
spectrogramThresholdDb={spectrogramThresholdDb}
|
||||
spectrogramPower={spectrogramPower}
|
||||
>
|
||||
{memoizedNotes}
|
||||
{recordingNoteOverlays}
|
||||
{!isSpectrogram && recordingNoteOverlays}
|
||||
</PianoGrid>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -4,6 +4,13 @@ import { KGDropdown } from '../common';
|
||||
import { KGPianoRollState } from '../../core/state/KGPianoRollState';
|
||||
import { KGCore } from '../../core/KGCore';
|
||||
|
||||
const POWER_OPTIONS = [
|
||||
{ label: 'Linear', value: '1.0' },
|
||||
{ label: '√ (default)', value: '0.5' },
|
||||
{ label: 'Mild', value: '0.4' },
|
||||
{ label: 'Strong', value: '0.3' },
|
||||
];
|
||||
|
||||
interface PianoRollToolbarProps {
|
||||
activeTool: 'pointer' | 'pencil';
|
||||
onToolSelect: (tool: 'pointer' | 'pencil') => void;
|
||||
@@ -17,6 +24,11 @@ interface PianoRollToolbarProps {
|
||||
chordGuide: string;
|
||||
onChordGuideChange: (value: string) => void;
|
||||
blinkButton?: string | null;
|
||||
mode?: 'midi-edit' | 'spectrogram';
|
||||
thresholdDb?: number;
|
||||
onThresholdChange?: (db: number) => void;
|
||||
power?: number;
|
||||
onPowerChange?: (power: number) => void;
|
||||
}
|
||||
|
||||
const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
|
||||
@@ -31,12 +43,19 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
|
||||
onModeChange,
|
||||
chordGuide,
|
||||
onChordGuideChange,
|
||||
blinkButton = null
|
||||
blinkButton = null,
|
||||
mode = 'midi-edit',
|
||||
thresholdDb = -25,
|
||||
onThresholdChange,
|
||||
power = 0.5,
|
||||
onPowerChange,
|
||||
}) => {
|
||||
const isSpectrogram = mode === 'spectrogram';
|
||||
|
||||
return (
|
||||
<div className="piano-roll-toolbar">
|
||||
{!isSpectrogram && (
|
||||
<div className="toolbar-left">
|
||||
{/* Left section with mode and chord guide dropdowns */}
|
||||
<KGDropdown
|
||||
options={Object.entries(KGCore.FUNCTIONAL_CHORDS_DATA).map(([id, data]) => ({ label: data.name, value: id }))}
|
||||
value={selectedMode}
|
||||
@@ -59,9 +78,10 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
|
||||
showValueAsLabel={true}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!isSpectrogram && (
|
||||
<div className="toolbar-center">
|
||||
{/* Center section with pointer and pencil tools */}
|
||||
<button
|
||||
className={`tool-button ${activeTool === 'pointer' ? 'active' : ''}`}
|
||||
onClick={() => onToolSelect('pointer')}
|
||||
@@ -77,9 +97,11 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
|
||||
<FaPencilAlt />
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="toolbar-right">
|
||||
{/* Right section with quantization options */}
|
||||
{!isSpectrogram && (
|
||||
<>
|
||||
<KGDropdown
|
||||
options={KGPianoRollState.SNAP_OPTIONS}
|
||||
value={snapping}
|
||||
@@ -88,7 +110,6 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
|
||||
buttonClassName="snapping"
|
||||
showValueAsLabel={true}
|
||||
/>
|
||||
|
||||
<KGDropdown
|
||||
options={KGPianoRollState.QUANT_POS_OPTIONS}
|
||||
value={quantPosition}
|
||||
@@ -96,7 +117,6 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
|
||||
label="Qua. Pos."
|
||||
buttonClassName={`quant-position ${blinkButton === 'quant-position' ? 'button-blink' : ''}`}
|
||||
/>
|
||||
|
||||
<KGDropdown
|
||||
options={KGPianoRollState.QUANT_LEN_OPTIONS}
|
||||
value={quantLength}
|
||||
@@ -104,6 +124,33 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
|
||||
label="Qua. Len."
|
||||
buttonClassName={`quant-length ${blinkButton === 'quant-length' ? 'button-blink' : ''}`}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
|
||||
{isSpectrogram && (
|
||||
<div className="spectrogram-toolbar-controls">
|
||||
<span className="spectrogram-control-label">Floor</span>
|
||||
<input
|
||||
type="range"
|
||||
className="spectrogram-threshold-slider"
|
||||
min={-50}
|
||||
max={-5}
|
||||
step={1}
|
||||
value={thresholdDb}
|
||||
onChange={e => onThresholdChange?.(parseInt(e.target.value))}
|
||||
title={`Noise floor: ${thresholdDb} dB`}
|
||||
/>
|
||||
<span className="spectrogram-threshold-value">{thresholdDb} dB</span>
|
||||
<KGDropdown
|
||||
options={POWER_OPTIONS}
|
||||
value={power.toString()}
|
||||
onChange={v => onPowerChange?.(parseFloat(v))}
|
||||
label="Curve"
|
||||
buttonClassName="curve-dropdown"
|
||||
showValueAsLabel={true}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
import React, { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import * as Tone from 'tone';
|
||||
import { KGAudioRegion } from '../../core/region/KGAudioRegion';
|
||||
import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
|
||||
import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage';
|
||||
import type { SpectrogramRequest, SpectrogramResult } from '../../workers/spectrogramWorker';
|
||||
|
||||
interface SpectrogramCanvasProps {
|
||||
audioRegion: KGAudioRegion;
|
||||
trackId: string;
|
||||
projectName: string;
|
||||
bpm: number;
|
||||
thresholdDb: number;
|
||||
power: number;
|
||||
}
|
||||
|
||||
const PITCH_BINS = 128;
|
||||
const HOP_SIZE = 1024;
|
||||
|
||||
// Piecewise-linear RGB colormap: black → dark blue → blue → purple → red → orange → yellow
|
||||
// Hue rotates 240°→300°→0°→60°, bypassing green entirely.
|
||||
const COLORMAP_STOPS: Array<[number, [number, number, number]]> = [
|
||||
[0.00, [ 0, 0, 0]],
|
||||
[0.15, [ 0, 0, 180]],
|
||||
[0.35, [ 0, 60, 255]],
|
||||
[0.55, [180, 0, 120]],
|
||||
[0.70, [255, 0, 0]],
|
||||
[0.85, [255, 140, 0]],
|
||||
[1.00, [255, 255, 0]],
|
||||
];
|
||||
|
||||
function hotColormap(v: number): [number, number, number] {
|
||||
v = Math.max(0, Math.min(1, v));
|
||||
for (let i = 0; i < COLORMAP_STOPS.length - 1; i++) {
|
||||
const [t0, c0] = COLORMAP_STOPS[i];
|
||||
const [t1, c1] = COLORMAP_STOPS[i + 1];
|
||||
if (v <= t1) {
|
||||
const t = (v - t0) / (t1 - t0);
|
||||
return [
|
||||
Math.round(c0[0] + t * (c1[0] - c0[0])),
|
||||
Math.round(c0[1] + t * (c1[1] - c0[1])),
|
||||
Math.round(c0[2] + t * (c1[2] - c0[2])),
|
||||
];
|
||||
}
|
||||
}
|
||||
return COLORMAP_STOPS[COLORMAP_STOPS.length - 1][1];
|
||||
}
|
||||
|
||||
const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
||||
audioRegion,
|
||||
trackId,
|
||||
projectName,
|
||||
bpm,
|
||||
thresholdDb,
|
||||
power,
|
||||
}) => {
|
||||
const canvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const workerRef = useRef<Worker | null>(null);
|
||||
|
||||
// Cache the raw linear result so threshold/power changes re-render without re-running FFT
|
||||
const rawResultRef = useRef<SpectrogramResult | null>(null);
|
||||
const sampleRateRef = useRef<number>(44100);
|
||||
const regionDurationRef = useRef<number>(0);
|
||||
|
||||
const renderSpectrogram = useCallback((
|
||||
result: SpectrogramResult,
|
||||
sampleRate: number,
|
||||
regionDurationSeconds: number,
|
||||
thresholdDb: number,
|
||||
power: number,
|
||||
) => {
|
||||
const canvas = canvasRef.current;
|
||||
if (!canvas) return;
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (!ctx) return;
|
||||
|
||||
const beatWidth =
|
||||
parseInt(getComputedStyle(document.documentElement).getPropertyValue('--region-grid-beat-width')) || 40;
|
||||
const noteHeight =
|
||||
parseInt(getComputedStyle(document.documentElement).getPropertyValue('--region-piano-key-height')) || 20;
|
||||
|
||||
const totalBeats = (regionDurationSeconds * bpm) / 60;
|
||||
const canvasWidth = Math.ceil(totalBeats * beatWidth);
|
||||
const canvasHeight = PITCH_BINS * noteHeight;
|
||||
|
||||
// Convert dB threshold to linear: values below this → black
|
||||
const linearThreshold = Math.pow(10, thresholdDb / 20);
|
||||
|
||||
// 1. Paint at natural STFT resolution onto an offscreen canvas (timeSteps × 128).
|
||||
// Row i = pitchIndex i = pitch (107 − i). Apply threshold then power curve.
|
||||
const offscreen = document.createElement('canvas');
|
||||
offscreen.width = result.timeSteps;
|
||||
offscreen.height = PITCH_BINS;
|
||||
const offCtx = offscreen.getContext('2d');
|
||||
if (!offCtx) return;
|
||||
|
||||
const imgData = offCtx.createImageData(result.timeSteps, PITCH_BINS);
|
||||
const pixels = imgData.data;
|
||||
|
||||
for (let row = 0; row < PITCH_BINS; row++) {
|
||||
const pitch = 107 - row;
|
||||
for (let col = 0; col < result.timeSteps; col++) {
|
||||
const idx = (row * result.timeSteps + col) * 4;
|
||||
pixels[idx + 3] = 255; // always opaque
|
||||
|
||||
if (pitch < 12 || pitch > 107) continue; // outside range → black
|
||||
|
||||
const raw = result.data[col * PITCH_BINS + pitch];
|
||||
|
||||
// Hard threshold: values below noise floor → 0 (black)
|
||||
// Re-scale surviving range to [0,1] then apply power curve
|
||||
const gated = raw < linearThreshold
|
||||
? 0
|
||||
: Math.pow((raw - linearThreshold) / (1 - linearThreshold), power);
|
||||
|
||||
if (gated <= 0) continue; // stays black
|
||||
|
||||
const [r, g, b] = hotColormap(gated);
|
||||
pixels[idx] = r;
|
||||
pixels[idx + 1] = g;
|
||||
pixels[idx + 2] = b;
|
||||
}
|
||||
}
|
||||
|
||||
offCtx.putImageData(imgData, 0, 0);
|
||||
|
||||
// 2. Stretch onto the full canvas — browser bilinear filter smooths between bins.
|
||||
canvas.width = canvasWidth;
|
||||
canvas.height = canvasHeight;
|
||||
ctx.imageSmoothingEnabled = true;
|
||||
ctx.imageSmoothingQuality = 'high';
|
||||
ctx.drawImage(offscreen, 0, 0, canvasWidth, canvasHeight);
|
||||
}, [bpm]);
|
||||
|
||||
// Re-render without re-running the worker when threshold or power changes
|
||||
useEffect(() => {
|
||||
if (rawResultRef.current) {
|
||||
renderSpectrogram(
|
||||
rawResultRef.current,
|
||||
sampleRateRef.current,
|
||||
regionDurationRef.current,
|
||||
thresholdDb,
|
||||
power,
|
||||
);
|
||||
}
|
||||
}, [thresholdDb, power, renderSpectrogram]);
|
||||
|
||||
// Load audio + run worker when the audio region itself changes
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
|
||||
const compute = async () => {
|
||||
setLoading(true);
|
||||
workerRef.current?.terminate();
|
||||
workerRef.current = null;
|
||||
|
||||
try {
|
||||
let audioBuffer: AudioBuffer | undefined = KGAudioInterface.instance().getAudioBuffer(
|
||||
trackId,
|
||||
audioRegion.getAudioFileId()
|
||||
);
|
||||
|
||||
if (!audioBuffer) {
|
||||
const arrayBuffer = await KGAudioFileStorage.loadAudioFile(
|
||||
projectName,
|
||||
audioRegion.getAudioFileId()
|
||||
);
|
||||
if (cancelled) return;
|
||||
const actx = Tone.getContext().rawContext as AudioContext;
|
||||
audioBuffer = await actx.decodeAudioData(arrayBuffer);
|
||||
}
|
||||
|
||||
if (cancelled) return;
|
||||
|
||||
const pcm = audioBuffer.getChannelData(0);
|
||||
const sampleRate = audioBuffer.sampleRate;
|
||||
const regionDurationSeconds = (audioRegion.getLength() * 60) / bpm;
|
||||
|
||||
const worker = new Worker(
|
||||
new URL('../../workers/spectrogramWorker.ts', import.meta.url),
|
||||
{ type: 'module' }
|
||||
);
|
||||
workerRef.current = worker;
|
||||
|
||||
worker.onmessage = (e: MessageEvent<SpectrogramResult>) => {
|
||||
if (cancelled) { worker.terminate(); return; }
|
||||
rawResultRef.current = e.data;
|
||||
sampleRateRef.current = sampleRate;
|
||||
regionDurationRef.current = regionDurationSeconds;
|
||||
renderSpectrogram(e.data, sampleRate, regionDurationSeconds, thresholdDb, power);
|
||||
setLoading(false);
|
||||
worker.terminate();
|
||||
workerRef.current = null;
|
||||
};
|
||||
|
||||
const request: SpectrogramRequest = {
|
||||
pcm: pcm.slice(0) as Float32Array,
|
||||
sampleRate,
|
||||
clipStartOffsetSeconds: audioRegion.getClipStartOffsetSeconds(),
|
||||
regionDurationSeconds,
|
||||
bpm,
|
||||
};
|
||||
worker.postMessage(request, [request.pcm.buffer]);
|
||||
} catch (err) {
|
||||
if (!cancelled) {
|
||||
console.error('SpectrogramCanvas: failed to compute spectrogram', err);
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
compute();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
workerRef.current?.terminate();
|
||||
workerRef.current = null;
|
||||
};
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [audioRegion, trackId, projectName, bpm]);
|
||||
|
||||
return (
|
||||
<>
|
||||
<canvas
|
||||
ref={canvasRef}
|
||||
style={{
|
||||
position: 'absolute',
|
||||
top: 0,
|
||||
left: 0,
|
||||
zIndex: 0,
|
||||
pointerEvents: 'none',
|
||||
}}
|
||||
/>
|
||||
{loading && (
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute',
|
||||
top: 0,
|
||||
left: 0,
|
||||
zIndex: 1,
|
||||
padding: '6px 10px',
|
||||
background: 'rgba(0,0,0,0.6)',
|
||||
color: '#aaa',
|
||||
fontSize: '11px',
|
||||
pointerEvents: 'none',
|
||||
}}
|
||||
>
|
||||
Computing spectrogram…
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default SpectrogramCanvas;
|
||||
@@ -119,6 +119,27 @@
|
||||
background: rgba(0, 0, 0, 0.35);
|
||||
}
|
||||
|
||||
.region-spectrogram-btn {
|
||||
position: absolute;
|
||||
top: 4px;
|
||||
left: 4px;
|
||||
background: rgba(0, 0, 0, 0.25);
|
||||
color: #fff;
|
||||
border: 1px solid rgba(255, 255, 255, 0.2);
|
||||
border-radius: 3px;
|
||||
padding: 2px;
|
||||
margin: 0;
|
||||
cursor: pointer;
|
||||
z-index: 2;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.region-spectrogram-btn:hover {
|
||||
background: rgba(0, 0, 0, 0.35);
|
||||
}
|
||||
|
||||
/* Instrument dropdown specific styles */
|
||||
.instrument-dropdown .quant-dropdown {
|
||||
min-width: 80px;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import React, { useState, useRef, useEffect } from 'react';
|
||||
import './Region.css';
|
||||
import { FaPencilAlt } from 'react-icons/fa';
|
||||
import { MdGraphicEq } from 'react-icons/md';
|
||||
import type { ResizeAction } from '../interfaces';
|
||||
import { REGION_CONSTANTS, DEBUG_MODE } from '../../constants';
|
||||
import { KGMidiRegion } from '../../core/region/KGMidiRegion';
|
||||
@@ -27,6 +28,8 @@ interface RegionItemProps {
|
||||
onClick?: (regionId: string) => void;
|
||||
// Explicit open piano roll action from header pencil icon
|
||||
onOpenPianoRoll?: (regionId: string) => void;
|
||||
// Open spectrogram viewer for audio regions
|
||||
onOpenSpectrogram?: (regionId: string) => void;
|
||||
// MIDI region data for rendering notes
|
||||
midiRegion?: KGMidiRegion;
|
||||
// Audio region data for rendering waveform
|
||||
@@ -49,6 +52,7 @@ const RegionItem: React.FC<RegionItemProps> = ({
|
||||
onDragEnd,
|
||||
onClick,
|
||||
onOpenPianoRoll,
|
||||
onOpenSpectrogram,
|
||||
midiRegion,
|
||||
audioRegion,
|
||||
audioBuffer
|
||||
@@ -576,6 +580,26 @@ const RegionItem: React.FC<RegionItemProps> = ({
|
||||
<FaPencilAlt size={10} />
|
||||
</button>
|
||||
)}
|
||||
{audioRegion && (
|
||||
<button
|
||||
className="region-spectrogram-btn"
|
||||
title="View melodic spectrogram"
|
||||
onMouseDown={(e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
}}
|
||||
onClick={(e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
if (onOpenSpectrogram) {
|
||||
onOpenSpectrogram(id);
|
||||
}
|
||||
}}
|
||||
aria-label="View spectrogram"
|
||||
>
|
||||
<MdGraphicEq size={10} />
|
||||
</button>
|
||||
)}
|
||||
<canvas ref={canvasRef} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -26,6 +26,7 @@ interface TrackGridItemProps {
|
||||
onRegionDragEnd?: (regionId: string, finalBarNumber: number, finalTrackIndex: number) => void;
|
||||
onRegionClick?: (regionId: string) => void;
|
||||
onOpenPianoRoll?: (regionId: string) => void;
|
||||
onOpenSpectrogram?: (regionId: string) => void;
|
||||
allTracks?: KGTrack[]; // Added to access all tracks for drag operations
|
||||
onKGOneClipDrop?: (e: React.DragEvent<HTMLDivElement>, trackIndex: number) => void;
|
||||
}
|
||||
@@ -47,6 +48,7 @@ const TrackGridItem: React.FC<TrackGridItemProps> = ({
|
||||
onRegionDragEnd,
|
||||
onRegionClick,
|
||||
onOpenPianoRoll,
|
||||
onOpenSpectrogram,
|
||||
allTracks,
|
||||
onKGOneClipDrop,
|
||||
}) => {
|
||||
@@ -566,6 +568,9 @@ const TrackGridItem: React.FC<TrackGridItemProps> = ({
|
||||
onRegionClick(regionId);
|
||||
}
|
||||
}}
|
||||
onOpenSpectrogram={audioRegion ? (regionId) => {
|
||||
onOpenSpectrogram?.(regionId);
|
||||
} : undefined}
|
||||
midiRegion={midiRegion}
|
||||
audioRegion={audioRegion}
|
||||
audioBuffer={audioBuffer}
|
||||
|
||||
@@ -30,6 +30,7 @@ interface TrackGridPanelProps {
|
||||
onRegionUpdated?: (regionId: string, updates: Partial<RegionUI>, expectedModelUpdates?: { startBeat: number, length: number }) => void;
|
||||
onRegionClick?: (regionId: string) => void;
|
||||
onOpenPianoRoll?: (regionId: string) => void;
|
||||
onOpenSpectrogram?: (regionId: string) => void;
|
||||
onExternalDropComplete?: (trackIndex: number, regionUI: RegionUI) => void;
|
||||
}
|
||||
|
||||
@@ -46,6 +47,7 @@ const TrackGridPanel: React.FC<TrackGridPanelProps> = ({
|
||||
onRegionUpdated,
|
||||
onRegionClick,
|
||||
onOpenPianoRoll,
|
||||
onOpenSpectrogram,
|
||||
onExternalDropComplete,
|
||||
}) => {
|
||||
const gridContainerRef = useRef<HTMLDivElement>(null);
|
||||
@@ -651,6 +653,7 @@ const TrackGridPanel: React.FC<TrackGridPanelProps> = ({
|
||||
onRegionDragEnd={handleRegionDragEnd}
|
||||
onRegionClick={handleRegionClick}
|
||||
onOpenPianoRoll={onOpenPianoRoll}
|
||||
onOpenSpectrogram={onOpenSpectrogram}
|
||||
allTracks={tracks}
|
||||
onKGOneClipDrop={handleExternalDrop}
|
||||
/>
|
||||
|
||||
@@ -78,6 +78,7 @@ interface ProjectState {
|
||||
// Piano roll state
|
||||
showPianoRoll: boolean;
|
||||
activeRegionId: string | null;
|
||||
pianoRollMode: 'midi-edit' | 'spectrogram';
|
||||
|
||||
// ChatBox state
|
||||
showChatBox: boolean;
|
||||
@@ -146,6 +147,8 @@ interface ProjectState {
|
||||
// Piano roll actions
|
||||
setShowPianoRoll: (show: boolean) => void;
|
||||
setActiveRegionId: (regionId: string | null) => void;
|
||||
openMidiPianoRoll: (regionId: string) => void;
|
||||
openSpectrogramViewer: (regionId: string) => void;
|
||||
|
||||
// Project state cleanup
|
||||
cleanupProjectState: () => void;
|
||||
@@ -304,6 +307,7 @@ export const useProjectStore = create<ProjectState>((set, get) => {
|
||||
// Initial piano roll state
|
||||
showPianoRoll: false,
|
||||
activeRegionId: null,
|
||||
pianoRollMode: 'midi-edit' as const,
|
||||
|
||||
// Initial ChatBox state
|
||||
showChatBox: initialChatBoxState,
|
||||
@@ -1033,6 +1037,14 @@ export const useProjectStore = create<ProjectState>((set, get) => {
|
||||
set({ activeRegionId: regionId });
|
||||
},
|
||||
|
||||
openMidiPianoRoll: (regionId: string) => {
|
||||
set({ showPianoRoll: true, activeRegionId: regionId, pianoRollMode: 'midi-edit' });
|
||||
},
|
||||
|
||||
openSpectrogramViewer: (regionId: string) => {
|
||||
set({ showPianoRoll: true, activeRegionId: regionId, pianoRollMode: 'spectrogram' });
|
||||
},
|
||||
|
||||
// Project state cleanup - used when starting new/loading projects
|
||||
cleanupProjectState: () => {
|
||||
// Close piano roll if it's visible
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import FFT from 'fft.js';
|
||||
|
||||
export interface SpectrogramRequest {
|
||||
pcm: Float32Array;
|
||||
sampleRate: number;
|
||||
clipStartOffsetSeconds: number;
|
||||
regionDurationSeconds: number;
|
||||
bpm: number;
|
||||
}
|
||||
|
||||
export interface SpectrogramResult {
|
||||
data: Float32Array; // [timeSteps × 128] row-major, row = time step, col = pitch 0-127
|
||||
timeSteps: number;
|
||||
}
|
||||
|
||||
const FFT_SIZE = 8192;
|
||||
const HOP_SIZE = 1024;
|
||||
const PITCH_BINS = 128;
|
||||
|
||||
function hannWindow(size: number): Float32Array {
|
||||
const w = new Float32Array(size);
|
||||
for (let i = 0; i < size; i++) {
|
||||
w[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / (size - 1)));
|
||||
}
|
||||
return w;
|
||||
}
|
||||
|
||||
self.onmessage = (e: MessageEvent<SpectrogramRequest>) => {
|
||||
const { pcm, sampleRate, clipStartOffsetSeconds, regionDurationSeconds } = e.data;
|
||||
|
||||
const startSample = Math.floor(clipStartOffsetSeconds * sampleRate);
|
||||
const endSample = Math.min(pcm.length, startSample + Math.ceil(regionDurationSeconds * sampleRate));
|
||||
const regionSamples = pcm.subarray(startSample, endSample);
|
||||
|
||||
const fft = new FFT(FFT_SIZE);
|
||||
const hann = hannWindow(FFT_SIZE);
|
||||
const complexOut = fft.createComplexArray() as number[];
|
||||
const inputPadded = new Float32Array(FFT_SIZE);
|
||||
|
||||
const totalHops = Math.max(1, Math.ceil((regionSamples.length - FFT_SIZE) / HOP_SIZE) + 1);
|
||||
const result = new Float32Array(totalHops * PITCH_BINS);
|
||||
|
||||
let maxVal = 0;
|
||||
|
||||
for (let hop = 0; hop < totalHops; hop++) {
|
||||
const offset = hop * HOP_SIZE;
|
||||
|
||||
// Fill windowed frame (zero-pad at end if needed)
|
||||
inputPadded.fill(0);
|
||||
const available = Math.min(FFT_SIZE, regionSamples.length - offset);
|
||||
for (let i = 0; i < available; i++) {
|
||||
inputPadded[i] = regionSamples[offset + i] * hann[i];
|
||||
}
|
||||
|
||||
fft.realTransform(complexOut, inputPadded as unknown as number[]);
|
||||
fft.completeSpectrum(complexOut);
|
||||
|
||||
// Max-pool magnitude into pitch bins: keep the loudest FFT bin per semitone,
|
||||
// rather than summing. Summing inflates every bin by how many FFT bins land there.
|
||||
const pitchRow = hop * PITCH_BINS;
|
||||
const numBins = FFT_SIZE / 2;
|
||||
|
||||
for (let bin = 1; bin < numBins; bin++) {
|
||||
const freq = (bin * sampleRate) / FFT_SIZE;
|
||||
if (freq < 20 || freq > 20000) continue;
|
||||
|
||||
// Convert frequency to MIDI pitch; clamp to piano roll range C0–B7 (MIDI 12–107)
|
||||
const pitch = Math.round(69 + 12 * Math.log2(freq / 440));
|
||||
if (pitch < 12 || pitch > 107) continue;
|
||||
|
||||
const re = complexOut[2 * bin];
|
||||
const im = complexOut[2 * bin + 1];
|
||||
const magnitude = Math.sqrt(re * re + im * im);
|
||||
|
||||
if (magnitude > result[pitchRow + pitch]) {
|
||||
result[pitchRow + pitch] = magnitude;
|
||||
}
|
||||
}
|
||||
|
||||
// Track global max for normalization
|
||||
for (let p = 0; p < PITCH_BINS; p++) {
|
||||
if (result[pitchRow + p] > maxVal) maxVal = result[pitchRow + p];
|
||||
}
|
||||
}
|
||||
|
||||
// Linear normalization only — threshold and power curve are applied in the canvas
|
||||
// renderer so changing them is instant (no need to re-run the FFT).
|
||||
if (maxVal > 0) {
|
||||
for (let i = 0; i < result.length; i++) {
|
||||
result[i] = result[i] / maxVal;
|
||||
}
|
||||
}
|
||||
|
||||
const response: SpectrogramResult = { data: result, timeSteps: totalHops };
|
||||
self.postMessage(response, [result.buffer]);
|
||||
};
|
||||
Reference in New Issue
Block a user