feat: added audio track spectrogram visualization feature
This commit is contained in:
Xiaohan-Tian
@@ -6,6 +6,8 @@ import { isModifierKeyPressed } from '../../util/osUtil';
|
||||
import { generatePianoGridBackground, getMatchingChordsForPitch } from '../../util/scaleUtil';
|
||||
import type { KeySignature } from '../../core/KGProject';
|
||||
import { KGPianoRollState } from '../../core/state/KGPianoRollState';
|
||||
import SpectrogramCanvas from './SpectrogramCanvas';
|
||||
import type { KGAudioRegion } from '../../core/region/KGAudioRegion';
|
||||
|
||||
interface PianoGridProps {
|
||||
gridRef: MutableRefObject<HTMLDivElement | null>;
|
||||
@@ -24,6 +26,12 @@ interface PianoGridProps {
|
||||
selectedMode: string;
|
||||
keySignature: KeySignature;
|
||||
chordGuide: string;
|
||||
audioRegion?: KGAudioRegion;
|
||||
trackId?: string;
|
||||
projectName?: string;
|
||||
bpm?: number;
|
||||
spectrogramThresholdDb?: number;
|
||||
spectrogramPower?: number;
|
||||
}
|
||||
|
||||
interface CursorPosition {
|
||||
@@ -44,7 +52,13 @@ const PianoGrid: React.FC<PianoGridProps> = ({
|
||||
regionStartBeat = 0,
|
||||
selectedMode,
|
||||
keySignature,
|
||||
chordGuide
|
||||
chordGuide,
|
||||
audioRegion,
|
||||
trackId,
|
||||
projectName,
|
||||
bpm = 120,
|
||||
spectrogramThresholdDb = -25,
|
||||
spectrogramPower = 0.5,
|
||||
}) => {
|
||||
const [cursorPosition, setCursorPosition] = useState<CursorPosition | null>(null);
|
||||
const [isModifierPressed, setIsModifierPressed] = useState(false);
|
||||
@@ -209,13 +223,25 @@ const PianoGrid: React.FC<PianoGridProps> = ({
|
||||
<div
|
||||
className={`piano-grid ${isModifierPressed ? 'pencil-cursor' : ''}`}
|
||||
ref={gridRef}
|
||||
style={{ backgroundImage }}
|
||||
style={{ backgroundImage: audioRegion ? undefined : backgroundImage }}
|
||||
onDoubleClick={onDoubleClick}
|
||||
onClick={onClick}
|
||||
onMouseDown={(e) => onMouseDown(e)}
|
||||
onMouseMove={handleMouseMove}
|
||||
onMouseLeave={handleMouseLeave}
|
||||
>
|
||||
{/* Spectrogram layer — rendered at z-index 0, behind all highlights and notes */}
|
||||
{audioRegion && trackId && projectName && (
|
||||
<SpectrogramCanvas
|
||||
audioRegion={audioRegion}
|
||||
trackId={trackId}
|
||||
projectName={projectName}
|
||||
bpm={bpm}
|
||||
thresholdDb={spectrogramThresholdDb}
|
||||
power={spectrogramPower}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Cursor Highlights */}
|
||||
{cursorPosition && (
|
||||
<>
|
||||
|
||||
@@ -96,6 +96,36 @@
|
||||
left: 0;
|
||||
}
|
||||
|
||||
/* Spectrogram toolbar controls */
|
||||
.spectrogram-toolbar-controls {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.spectrogram-control-label {
|
||||
font-size: 10px;
|
||||
color: #aaa;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.spectrogram-threshold-slider {
|
||||
/* Match the visual width of the Qua. Pos. / Qua. Len. buttons (~80px) */
|
||||
width: 80px;
|
||||
height: 4px;
|
||||
accent-color: #7a9ccf;
|
||||
cursor: pointer;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.spectrogram-threshold-value {
|
||||
font-size: 10px;
|
||||
color: #e0e0e0;
|
||||
min-width: 44px;
|
||||
text-align: right;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.piano-roll-title {
|
||||
flex: 1;
|
||||
text-align: center;
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { MouseEvent } from 'react';
|
||||
import { useProjectStore } from '../../stores/projectStore';
|
||||
import { FaGripLines } from 'react-icons/fa';
|
||||
import { KGMidiRegion } from '../../core/region/KGMidiRegion';
|
||||
import type { KGAudioRegion } from '../../core/region/KGAudioRegion';
|
||||
import { DEBUG_MODE, PIANO_ROLL_CONSTANTS } from '../../constants';
|
||||
import PianoRollHeader from './PianoRollHeader';
|
||||
import PianoRollToolbar from './PianoRollToolbar';
|
||||
@@ -22,18 +23,31 @@ interface PianoRollProps {
|
||||
regionId: string | null;
|
||||
initialPosition?: { x: number; y: number };
|
||||
initialSize?: { width: number; height: number };
|
||||
mode?: 'midi-edit' | 'spectrogram';
|
||||
audioRegion?: KGAudioRegion;
|
||||
trackId?: string;
|
||||
projectName?: string;
|
||||
}
|
||||
|
||||
const PianoRoll: React.FC<PianoRollProps> = ({
|
||||
onClose,
|
||||
regionId,
|
||||
initialPosition,
|
||||
initialSize
|
||||
initialSize,
|
||||
mode = 'midi-edit',
|
||||
audioRegion,
|
||||
trackId,
|
||||
projectName,
|
||||
}) => {
|
||||
const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled } = useProjectStore();
|
||||
const isSpectrogram = mode === 'spectrogram';
|
||||
const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled, bpm } = useProjectStore();
|
||||
|
||||
// Tool state for piano roll
|
||||
const [activeTool, setActiveTool] = useState<'pointer' | 'pencil'>('pointer');
|
||||
|
||||
// Spectrogram controls (only used in spectrogram mode)
|
||||
const [spectrogramThresholdDb, setSpectrogramThresholdDb] = useState<number>(-25);
|
||||
const [spectrogramPower, setSpectrogramPower] = useState<number>(0.5);
|
||||
|
||||
// Quantization state
|
||||
const [quantPosition, setQuantPosition] = useState<string>('1/8');
|
||||
@@ -855,6 +869,7 @@ const PianoRoll: React.FC<PianoRollProps> = ({
|
||||
|
||||
// Get the title for the piano roll based on the active region
|
||||
const getPianoRollTitle = () => {
|
||||
if (isSpectrogram) return audioRegion ? `SPECTROGRAM — ${audioRegion.getName()}` : 'SPECTROGRAM';
|
||||
if (!activeRegion) return "EDIT NOTE CLIP";
|
||||
|
||||
// Calculate the bar and beat position of the region
|
||||
@@ -901,8 +916,13 @@ const PianoRoll: React.FC<PianoRollProps> = ({
|
||||
chordGuide={chordGuide}
|
||||
onChordGuideChange={handleChordGuideSelect}
|
||||
blinkButton={blinkButton}
|
||||
mode={mode}
|
||||
thresholdDb={spectrogramThresholdDb}
|
||||
onThresholdChange={setSpectrogramThresholdDb}
|
||||
power={spectrogramPower}
|
||||
onPowerChange={setSpectrogramPower}
|
||||
/>
|
||||
|
||||
|
||||
<PianoRollContent
|
||||
contentRef={pianoRollContentRef}
|
||||
pianoGridRef={pianoGridRef}
|
||||
@@ -916,6 +936,13 @@ const PianoRoll: React.FC<PianoRollProps> = ({
|
||||
selectedMode={selectedMode}
|
||||
keySignature={keySignature}
|
||||
chordGuide={chordGuide}
|
||||
mode={mode}
|
||||
audioRegion={audioRegion}
|
||||
trackId={trackId}
|
||||
projectName={projectName}
|
||||
bpm={bpm}
|
||||
spectrogramThresholdDb={spectrogramThresholdDb}
|
||||
spectrogramPower={spectrogramPower}
|
||||
/>
|
||||
|
||||
<div
|
||||
|
||||
@@ -12,6 +12,7 @@ import PianoGrid from './PianoGrid';
|
||||
import { useNoteOperations } from '../../hooks/useNoteOperations';
|
||||
import { useNoteSelection } from '../../hooks/useNoteSelection';
|
||||
import type { KeySignature } from '../../core/KGProject';
|
||||
import type { KGAudioRegion } from '../../core/region/KGAudioRegion';
|
||||
|
||||
interface PianoRollContentProps {
|
||||
contentRef: React.MutableRefObject<HTMLDivElement | null>;
|
||||
@@ -26,6 +27,13 @@ interface PianoRollContentProps {
|
||||
selectedMode: string;
|
||||
keySignature: KeySignature;
|
||||
chordGuide: string;
|
||||
mode?: 'midi-edit' | 'spectrogram';
|
||||
audioRegion?: KGAudioRegion;
|
||||
trackId?: string;
|
||||
projectName?: string;
|
||||
bpm?: number;
|
||||
spectrogramThresholdDb?: number;
|
||||
spectrogramPower?: number;
|
||||
}
|
||||
|
||||
const PianoRollContent: React.FC<PianoRollContentProps> = ({
|
||||
@@ -40,8 +48,16 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
|
||||
onSetDeleteNotesTrigger,
|
||||
selectedMode,
|
||||
keySignature,
|
||||
chordGuide
|
||||
chordGuide,
|
||||
mode = 'midi-edit',
|
||||
audioRegion,
|
||||
trackId,
|
||||
projectName,
|
||||
bpm = 120,
|
||||
spectrogramThresholdDb = -25,
|
||||
spectrogramPower = 0.5,
|
||||
}) => {
|
||||
const isSpectrogram = mode === 'spectrogram';
|
||||
// Get KGCore instance
|
||||
const core = KGCore.instance();
|
||||
|
||||
@@ -105,9 +121,8 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
|
||||
|
||||
// Combined click handler for both pointer and pencil modes
|
||||
const handleCombinedClick = (e: React.MouseEvent) => {
|
||||
// Handle selection click (pointer mode)
|
||||
if (isSpectrogram) return;
|
||||
handleBackgroundClick(e);
|
||||
// Handle pencil mode note creation
|
||||
handleGridClick(e);
|
||||
};
|
||||
|
||||
@@ -144,7 +159,7 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
|
||||
|
||||
// Memoize the notes rendering to prevent unnecessary recalculations
|
||||
const memoizedNotes = useMemo(() => {
|
||||
if (!activeRegion) return null;
|
||||
if (isSpectrogram || !activeRegion) return null;
|
||||
|
||||
if (DEBUG_MODE.PIANO_ROLL) {
|
||||
console.log(`Rendering notes for region: ${activeRegion.getId()}`);
|
||||
@@ -251,18 +266,24 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
|
||||
|
||||
<PianoGrid
|
||||
gridRef={pianoGridRef}
|
||||
onDoubleClick={handleGridDoubleClick}
|
||||
onClick={handleCombinedClick}
|
||||
onMouseDown={handleBackgroundMouseDown}
|
||||
isBoxSelecting={isBoxSelectingRef.current}
|
||||
selectionBox={selectionBoxRef.current}
|
||||
onDoubleClick={isSpectrogram ? () => {} : handleGridDoubleClick}
|
||||
onClick={isSpectrogram ? () => {} : handleCombinedClick}
|
||||
onMouseDown={isSpectrogram ? () => {} : handleBackgroundMouseDown}
|
||||
isBoxSelecting={isSpectrogram ? false : isBoxSelectingRef.current}
|
||||
selectionBox={isSpectrogram ? { startX: 0, startY: 0, endX: 0, endY: 0 } : selectionBoxRef.current}
|
||||
regionStartBeat={activeRegion?.getStartFromBeat() || 0}
|
||||
selectedMode={selectedMode}
|
||||
keySignature={keySignature}
|
||||
chordGuide={chordGuide}
|
||||
audioRegion={audioRegion}
|
||||
trackId={trackId}
|
||||
projectName={projectName}
|
||||
bpm={bpm}
|
||||
spectrogramThresholdDb={spectrogramThresholdDb}
|
||||
spectrogramPower={spectrogramPower}
|
||||
>
|
||||
{memoizedNotes}
|
||||
{recordingNoteOverlays}
|
||||
{!isSpectrogram && recordingNoteOverlays}
|
||||
</PianoGrid>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -4,6 +4,13 @@ import { KGDropdown } from '../common';
|
||||
import { KGPianoRollState } from '../../core/state/KGPianoRollState';
|
||||
import { KGCore } from '../../core/KGCore';
|
||||
|
||||
const POWER_OPTIONS = [
|
||||
{ label: 'Linear', value: '1.0' },
|
||||
{ label: '√ (default)', value: '0.5' },
|
||||
{ label: 'Mild', value: '0.4' },
|
||||
{ label: 'Strong', value: '0.3' },
|
||||
];
|
||||
|
||||
interface PianoRollToolbarProps {
|
||||
activeTool: 'pointer' | 'pencil';
|
||||
onToolSelect: (tool: 'pointer' | 'pencil') => void;
|
||||
@@ -17,6 +24,11 @@ interface PianoRollToolbarProps {
|
||||
chordGuide: string;
|
||||
onChordGuideChange: (value: string) => void;
|
||||
blinkButton?: string | null;
|
||||
mode?: 'midi-edit' | 'spectrogram';
|
||||
thresholdDb?: number;
|
||||
onThresholdChange?: (db: number) => void;
|
||||
power?: number;
|
||||
onPowerChange?: (power: number) => void;
|
||||
}
|
||||
|
||||
const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
|
||||
@@ -31,82 +43,117 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
|
||||
onModeChange,
|
||||
chordGuide,
|
||||
onChordGuideChange,
|
||||
blinkButton = null
|
||||
blinkButton = null,
|
||||
mode = 'midi-edit',
|
||||
thresholdDb = -25,
|
||||
onThresholdChange,
|
||||
power = 0.5,
|
||||
onPowerChange,
|
||||
}) => {
|
||||
const isSpectrogram = mode === 'spectrogram';
|
||||
|
||||
return (
|
||||
<div className="piano-roll-toolbar">
|
||||
<div className="toolbar-left">
|
||||
{/* Left section with mode and chord guide dropdowns */}
|
||||
<KGDropdown
|
||||
options={Object.entries(KGCore.FUNCTIONAL_CHORDS_DATA).map(([id, data]) => ({ label: data.name, value: id }))}
|
||||
value={selectedMode}
|
||||
onChange={(value) => onModeChange(value)}
|
||||
label="Mode"
|
||||
buttonClassName="mode-dropdown"
|
||||
showValueAsLabel={true}
|
||||
/>
|
||||
<KGDropdown
|
||||
options={[
|
||||
{ label: 'Guide: Disabled', value: 'N' },
|
||||
{ label: 'Chord Guide: T', value: 'T' },
|
||||
{ label: 'Chord Guide: S', value: 'S' },
|
||||
{ label: 'Chord Guide: D', value: 'D' }
|
||||
]}
|
||||
value={chordGuide}
|
||||
onChange={(value) => onChordGuideChange(value)}
|
||||
label="Chord"
|
||||
buttonClassName="chord-guide-dropdown"
|
||||
showValueAsLabel={true}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="toolbar-center">
|
||||
{/* Center section with pointer and pencil tools */}
|
||||
<button
|
||||
className={`tool-button ${activeTool === 'pointer' ? 'active' : ''}`}
|
||||
onClick={() => onToolSelect('pointer')}
|
||||
title="Pointer Tool"
|
||||
>
|
||||
<FaMousePointer />
|
||||
</button>
|
||||
<button
|
||||
className={`tool-button ${activeTool === 'pencil' ? 'active' : ''}`}
|
||||
onClick={() => onToolSelect('pencil')}
|
||||
title="Pencil Tool"
|
||||
>
|
||||
<FaPencilAlt />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{!isSpectrogram && (
|
||||
<div className="toolbar-left">
|
||||
<KGDropdown
|
||||
options={Object.entries(KGCore.FUNCTIONAL_CHORDS_DATA).map(([id, data]) => ({ label: data.name, value: id }))}
|
||||
value={selectedMode}
|
||||
onChange={(value) => onModeChange(value)}
|
||||
label="Mode"
|
||||
buttonClassName="mode-dropdown"
|
||||
showValueAsLabel={true}
|
||||
/>
|
||||
<KGDropdown
|
||||
options={[
|
||||
{ label: 'Guide: Disabled', value: 'N' },
|
||||
{ label: 'Chord Guide: T', value: 'T' },
|
||||
{ label: 'Chord Guide: S', value: 'S' },
|
||||
{ label: 'Chord Guide: D', value: 'D' }
|
||||
]}
|
||||
value={chordGuide}
|
||||
onChange={(value) => onChordGuideChange(value)}
|
||||
label="Chord"
|
||||
buttonClassName="chord-guide-dropdown"
|
||||
showValueAsLabel={true}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!isSpectrogram && (
|
||||
<div className="toolbar-center">
|
||||
<button
|
||||
className={`tool-button ${activeTool === 'pointer' ? 'active' : ''}`}
|
||||
onClick={() => onToolSelect('pointer')}
|
||||
title="Pointer Tool"
|
||||
>
|
||||
<FaMousePointer />
|
||||
</button>
|
||||
<button
|
||||
className={`tool-button ${activeTool === 'pencil' ? 'active' : ''}`}
|
||||
onClick={() => onToolSelect('pencil')}
|
||||
title="Pencil Tool"
|
||||
>
|
||||
<FaPencilAlt />
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="toolbar-right">
|
||||
{/* Right section with quantization options */}
|
||||
<KGDropdown
|
||||
options={KGPianoRollState.SNAP_OPTIONS}
|
||||
value={snapping}
|
||||
onChange={(value) => onSnappingSelect(value)}
|
||||
label="Snap"
|
||||
buttonClassName="snapping"
|
||||
showValueAsLabel={true}
|
||||
/>
|
||||
{!isSpectrogram && (
|
||||
<>
|
||||
<KGDropdown
|
||||
options={KGPianoRollState.SNAP_OPTIONS}
|
||||
value={snapping}
|
||||
onChange={(value) => onSnappingSelect(value)}
|
||||
label="Snap"
|
||||
buttonClassName="snapping"
|
||||
showValueAsLabel={true}
|
||||
/>
|
||||
<KGDropdown
|
||||
options={KGPianoRollState.QUANT_POS_OPTIONS}
|
||||
value={quantPosition}
|
||||
onChange={(value) => onQuantSelect('position', value)}
|
||||
label="Qua. Pos."
|
||||
buttonClassName={`quant-position ${blinkButton === 'quant-position' ? 'button-blink' : ''}`}
|
||||
/>
|
||||
<KGDropdown
|
||||
options={KGPianoRollState.QUANT_LEN_OPTIONS}
|
||||
value={quantLength}
|
||||
onChange={(value) => onQuantSelect('length', value)}
|
||||
label="Qua. Len."
|
||||
buttonClassName={`quant-length ${blinkButton === 'quant-length' ? 'button-blink' : ''}`}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
|
||||
<KGDropdown
|
||||
options={KGPianoRollState.QUANT_POS_OPTIONS}
|
||||
value={quantPosition}
|
||||
onChange={(value) => onQuantSelect('position', value)}
|
||||
label="Qua. Pos."
|
||||
buttonClassName={`quant-position ${blinkButton === 'quant-position' ? 'button-blink' : ''}`}
|
||||
/>
|
||||
|
||||
<KGDropdown
|
||||
options={KGPianoRollState.QUANT_LEN_OPTIONS}
|
||||
value={quantLength}
|
||||
onChange={(value) => onQuantSelect('length', value)}
|
||||
label="Qua. Len."
|
||||
buttonClassName={`quant-length ${blinkButton === 'quant-length' ? 'button-blink' : ''}`}
|
||||
/>
|
||||
{isSpectrogram && (
|
||||
<div className="spectrogram-toolbar-controls">
|
||||
<span className="spectrogram-control-label">Floor</span>
|
||||
<input
|
||||
type="range"
|
||||
className="spectrogram-threshold-slider"
|
||||
min={-50}
|
||||
max={-5}
|
||||
step={1}
|
||||
value={thresholdDb}
|
||||
onChange={e => onThresholdChange?.(parseInt(e.target.value))}
|
||||
title={`Noise floor: ${thresholdDb} dB`}
|
||||
/>
|
||||
<span className="spectrogram-threshold-value">{thresholdDb} dB</span>
|
||||
<KGDropdown
|
||||
options={POWER_OPTIONS}
|
||||
value={power.toString()}
|
||||
onChange={v => onPowerChange?.(parseFloat(v))}
|
||||
label="Curve"
|
||||
buttonClassName="curve-dropdown"
|
||||
showValueAsLabel={true}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default PianoRollToolbar;
|
||||
export default PianoRollToolbar;
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
import React, { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import * as Tone from 'tone';
|
||||
import { KGAudioRegion } from '../../core/region/KGAudioRegion';
|
||||
import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
|
||||
import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage';
|
||||
import type { SpectrogramRequest, SpectrogramResult } from '../../workers/spectrogramWorker';
|
||||
|
||||
interface SpectrogramCanvasProps {
|
||||
audioRegion: KGAudioRegion;
|
||||
trackId: string;
|
||||
projectName: string;
|
||||
bpm: number;
|
||||
thresholdDb: number;
|
||||
power: number;
|
||||
}
|
||||
|
||||
const PITCH_BINS = 128;
|
||||
const HOP_SIZE = 1024;
|
||||
|
||||
// Piecewise-linear RGB colormap: black → dark blue → blue → purple → red → orange → yellow
|
||||
// Hue rotates 240°→300°→0°→60°, bypassing green entirely.
|
||||
const COLORMAP_STOPS: Array<[number, [number, number, number]]> = [
|
||||
[0.00, [ 0, 0, 0]],
|
||||
[0.15, [ 0, 0, 180]],
|
||||
[0.35, [ 0, 60, 255]],
|
||||
[0.55, [180, 0, 120]],
|
||||
[0.70, [255, 0, 0]],
|
||||
[0.85, [255, 140, 0]],
|
||||
[1.00, [255, 255, 0]],
|
||||
];
|
||||
|
||||
function hotColormap(v: number): [number, number, number] {
|
||||
v = Math.max(0, Math.min(1, v));
|
||||
for (let i = 0; i < COLORMAP_STOPS.length - 1; i++) {
|
||||
const [t0, c0] = COLORMAP_STOPS[i];
|
||||
const [t1, c1] = COLORMAP_STOPS[i + 1];
|
||||
if (v <= t1) {
|
||||
const t = (v - t0) / (t1 - t0);
|
||||
return [
|
||||
Math.round(c0[0] + t * (c1[0] - c0[0])),
|
||||
Math.round(c0[1] + t * (c1[1] - c0[1])),
|
||||
Math.round(c0[2] + t * (c1[2] - c0[2])),
|
||||
];
|
||||
}
|
||||
}
|
||||
return COLORMAP_STOPS[COLORMAP_STOPS.length - 1][1];
|
||||
}
|
||||
|
||||
const SpectrogramCanvas: React.FC<SpectrogramCanvasProps> = ({
|
||||
audioRegion,
|
||||
trackId,
|
||||
projectName,
|
||||
bpm,
|
||||
thresholdDb,
|
||||
power,
|
||||
}) => {
|
||||
const canvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const workerRef = useRef<Worker | null>(null);
|
||||
|
||||
// Cache the raw linear result so threshold/power changes re-render without re-running FFT
|
||||
const rawResultRef = useRef<SpectrogramResult | null>(null);
|
||||
const sampleRateRef = useRef<number>(44100);
|
||||
const regionDurationRef = useRef<number>(0);
|
||||
|
||||
const renderSpectrogram = useCallback((
|
||||
result: SpectrogramResult,
|
||||
sampleRate: number,
|
||||
regionDurationSeconds: number,
|
||||
thresholdDb: number,
|
||||
power: number,
|
||||
) => {
|
||||
const canvas = canvasRef.current;
|
||||
if (!canvas) return;
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (!ctx) return;
|
||||
|
||||
const beatWidth =
|
||||
parseInt(getComputedStyle(document.documentElement).getPropertyValue('--region-grid-beat-width')) || 40;
|
||||
const noteHeight =
|
||||
parseInt(getComputedStyle(document.documentElement).getPropertyValue('--region-piano-key-height')) || 20;
|
||||
|
||||
const totalBeats = (regionDurationSeconds * bpm) / 60;
|
||||
const canvasWidth = Math.ceil(totalBeats * beatWidth);
|
||||
const canvasHeight = PITCH_BINS * noteHeight;
|
||||
|
||||
// Convert dB threshold to linear: values below this → black
|
||||
const linearThreshold = Math.pow(10, thresholdDb / 20);
|
||||
|
||||
// 1. Paint at natural STFT resolution onto an offscreen canvas (timeSteps × 128).
|
||||
// Row i = pitchIndex i = pitch (107 − i). Apply threshold then power curve.
|
||||
const offscreen = document.createElement('canvas');
|
||||
offscreen.width = result.timeSteps;
|
||||
offscreen.height = PITCH_BINS;
|
||||
const offCtx = offscreen.getContext('2d');
|
||||
if (!offCtx) return;
|
||||
|
||||
const imgData = offCtx.createImageData(result.timeSteps, PITCH_BINS);
|
||||
const pixels = imgData.data;
|
||||
|
||||
for (let row = 0; row < PITCH_BINS; row++) {
|
||||
const pitch = 107 - row;
|
||||
for (let col = 0; col < result.timeSteps; col++) {
|
||||
const idx = (row * result.timeSteps + col) * 4;
|
||||
pixels[idx + 3] = 255; // always opaque
|
||||
|
||||
if (pitch < 12 || pitch > 107) continue; // outside range → black
|
||||
|
||||
const raw = result.data[col * PITCH_BINS + pitch];
|
||||
|
||||
// Hard threshold: values below noise floor → 0 (black)
|
||||
// Re-scale surviving range to [0,1] then apply power curve
|
||||
const gated = raw < linearThreshold
|
||||
? 0
|
||||
: Math.pow((raw - linearThreshold) / (1 - linearThreshold), power);
|
||||
|
||||
if (gated <= 0) continue; // stays black
|
||||
|
||||
const [r, g, b] = hotColormap(gated);
|
||||
pixels[idx] = r;
|
||||
pixels[idx + 1] = g;
|
||||
pixels[idx + 2] = b;
|
||||
}
|
||||
}
|
||||
|
||||
offCtx.putImageData(imgData, 0, 0);
|
||||
|
||||
// 2. Stretch onto the full canvas — browser bilinear filter smooths between bins.
|
||||
canvas.width = canvasWidth;
|
||||
canvas.height = canvasHeight;
|
||||
ctx.imageSmoothingEnabled = true;
|
||||
ctx.imageSmoothingQuality = 'high';
|
||||
ctx.drawImage(offscreen, 0, 0, canvasWidth, canvasHeight);
|
||||
}, [bpm]);
|
||||
|
||||
// Re-render without re-running the worker when threshold or power changes
|
||||
useEffect(() => {
|
||||
if (rawResultRef.current) {
|
||||
renderSpectrogram(
|
||||
rawResultRef.current,
|
||||
sampleRateRef.current,
|
||||
regionDurationRef.current,
|
||||
thresholdDb,
|
||||
power,
|
||||
);
|
||||
}
|
||||
}, [thresholdDb, power, renderSpectrogram]);
|
||||
|
||||
// Load audio + run worker when the audio region itself changes
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
|
||||
const compute = async () => {
|
||||
setLoading(true);
|
||||
workerRef.current?.terminate();
|
||||
workerRef.current = null;
|
||||
|
||||
try {
|
||||
let audioBuffer: AudioBuffer | undefined = KGAudioInterface.instance().getAudioBuffer(
|
||||
trackId,
|
||||
audioRegion.getAudioFileId()
|
||||
);
|
||||
|
||||
if (!audioBuffer) {
|
||||
const arrayBuffer = await KGAudioFileStorage.loadAudioFile(
|
||||
projectName,
|
||||
audioRegion.getAudioFileId()
|
||||
);
|
||||
if (cancelled) return;
|
||||
const actx = Tone.getContext().rawContext as AudioContext;
|
||||
audioBuffer = await actx.decodeAudioData(arrayBuffer);
|
||||
}
|
||||
|
||||
if (cancelled) return;
|
||||
|
||||
const pcm = audioBuffer.getChannelData(0);
|
||||
const sampleRate = audioBuffer.sampleRate;
|
||||
const regionDurationSeconds = (audioRegion.getLength() * 60) / bpm;
|
||||
|
||||
const worker = new Worker(
|
||||
new URL('../../workers/spectrogramWorker.ts', import.meta.url),
|
||||
{ type: 'module' }
|
||||
);
|
||||
workerRef.current = worker;
|
||||
|
||||
worker.onmessage = (e: MessageEvent<SpectrogramResult>) => {
|
||||
if (cancelled) { worker.terminate(); return; }
|
||||
rawResultRef.current = e.data;
|
||||
sampleRateRef.current = sampleRate;
|
||||
regionDurationRef.current = regionDurationSeconds;
|
||||
renderSpectrogram(e.data, sampleRate, regionDurationSeconds, thresholdDb, power);
|
||||
setLoading(false);
|
||||
worker.terminate();
|
||||
workerRef.current = null;
|
||||
};
|
||||
|
||||
const request: SpectrogramRequest = {
|
||||
pcm: pcm.slice(0) as Float32Array,
|
||||
sampleRate,
|
||||
clipStartOffsetSeconds: audioRegion.getClipStartOffsetSeconds(),
|
||||
regionDurationSeconds,
|
||||
bpm,
|
||||
};
|
||||
worker.postMessage(request, [request.pcm.buffer]);
|
||||
} catch (err) {
|
||||
if (!cancelled) {
|
||||
console.error('SpectrogramCanvas: failed to compute spectrogram', err);
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
compute();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
workerRef.current?.terminate();
|
||||
workerRef.current = null;
|
||||
};
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [audioRegion, trackId, projectName, bpm]);
|
||||
|
||||
return (
|
||||
<>
|
||||
<canvas
|
||||
ref={canvasRef}
|
||||
style={{
|
||||
position: 'absolute',
|
||||
top: 0,
|
||||
left: 0,
|
||||
zIndex: 0,
|
||||
pointerEvents: 'none',
|
||||
}}
|
||||
/>
|
||||
{loading && (
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute',
|
||||
top: 0,
|
||||
left: 0,
|
||||
zIndex: 1,
|
||||
padding: '6px 10px',
|
||||
background: 'rgba(0,0,0,0.6)',
|
||||
color: '#aaa',
|
||||
fontSize: '11px',
|
||||
pointerEvents: 'none',
|
||||
}}
|
||||
>
|
||||
Computing spectrogram…
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default SpectrogramCanvas;
|
||||
Reference in New Issue
Block a user