3dtours/KGStudio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add audio chord detection for spectrogram regions

Browse Source
This commit is contained in:
Xiaohan-Tian
2026-05-25 19:09:19 -07:00
parent a7158b1ee0
commit f12add935a
15 changed files with 931 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/piano-roll/PianoRoll.css
+35 -4
View File
@@ -235,6 +235,11 @@
left: 0;
}
.piano-roll-toolbar .quant-option.disabled {
color: #666;
cursor: default;
}
.piano-roll-automation-toolbar-group {
display: flex;
align-items: center;
@@ -408,11 +413,37 @@
}
.spectrogram-loading-label {
padding: 6px 10px;
background: rgba(0, 0, 0, 0.6);
color: #aaa;
min-width: 240px;
padding: 10px 12px;
background: rgba(0, 0, 0, 0.72);
color: #ddd;
font-size: 11px;
border-radius: 3px;
border: 1px solid #3a3a3a;
border-radius: 6px;
display: flex;
flex-direction: column;
gap: 8px;
}
.piano-roll-progress-block {
display: flex;
flex-direction: column;
gap: 6px;
}
.piano-roll-progress-track {
width: 100%;
height: 8px;
background-color: #1d1d1d;
border: 1px solid #3a3a3a;
border-radius: 999px;
overflow: hidden;
}
.piano-roll-progress-fill {
height: 100%;
background: linear-gradient(90deg, #5a9fd4 0%, #7cc2f1 100%);
transition: width 0.15s linear;
}
.piano-grid-header {
src/components/piano-roll/PianoRoll.tsx
+103 -2
View File
@@ -1,5 +1,6 @@
import React, { useRef, useEffect, useState, useCallback, useLayoutEffect, useMemo } from 'react';
import './PianoRoll.css';
import * as Tone from 'tone';
import { useProjectStore } from '../../stores/projectStore';
import { FaGripLines } from 'react-icons/fa';
import { KGMidiRegion } from '../../core/region/KGMidiRegion';
@@ -15,13 +16,21 @@ import { KGMidiTrack, type InstrumentType } from '../../core/track/KGMidiTrack';
import { KGPianoRollState } from '../../core/state/KGPianoRollState';
import { ConfigManager } from '../../core/config/ConfigManager';
import { beatsToBar } from '../../util/midiUtil';
import { UpdateRegionCommand } from '../../core/commands';
import { ReplaceChordRegionsInRangeCommand, UpdateRegionCommand } from '../../core/commands';
import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage';
import { getSuitableChords, noteNameToPitchClass } from '../../util/scaleUtil';
import { showAlert } from '../../util/dialogUtil';
import {
normalizeSpectrogramHeightResolution,
type SpectrogramHeightResolution,
} from '../../util/spectrogramUtil';
import {
buildAudioChordWindowsForRegion,
type AudioChordDetectionRequest,
type DetectedAudioChord,
} from '../../util/audioChordDetection';
import type { AudioChordDetectionWorkerMessage } from '../../workers/audioChordDetectionWorker';
import type { PianoRollAutomationType } from './pianoRollAutomation';
import type { SheetMeasureMetric } from './sheetNotationTypes';
import { getSheetPlayheadPixel, getSheetQuantizationOptions, parseSheetQuantization } from './sheetNotation';
@@ -59,7 +68,7 @@ const PianoRoll: React.FC<PianoRollProps> = ({
}) => {
const isSpectrogram = mode === 'spectrogram';
const isHybrid = mode === 'hybrid';
const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showKGOnePanel, showEventListPanel, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled, bpm, pianoRollScrollRequest, selectedNoteIds, automationRedrawVersion } = useProjectStore();
const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showKGOnePanel, showEventListPanel, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled, bpm, pianoRollScrollRequest, selectedNoteIds, automationRedrawVersion, refreshProjectState } = useProjectStore();
// Tool state for piano roll
const [activeTool, setActiveTool] = useState<'pointer' | 'pencil'>('pointer');
@@ -69,6 +78,8 @@ const PianoRoll: React.FC<PianoRollProps> = ({
const [spectrogramPower, setSpectrogramPower] = useState<number>(0.5);
const [spectrogramHeightResolution, setSpectrogramHeightResolution] =
useState<SpectrogramHeightResolution>(3);
const [isDetectingChords, setIsDetectingChords] = useState(false);
const [detectChordProgressPercent, setDetectChordProgressPercent] = useState(0);
// Piano roll zoom (1x–8x); updates --region-grid-beat-width CSS variable
const [pianoRollZoom, setPianoRollZoom] = useState<number>(() => KGPianoRollState.instance().getPianoRollZoom());
@@ -436,6 +447,92 @@ const PianoRoll: React.FC<PianoRollProps> = ({
}
};
const handleDetectChords = useCallback(async () => {
if (!audioRegion || !projectName || !trackId) {
await showAlert('Open an audio region in spectrogram mode before detecting chords.');
return;
}
const project = KGCore.instance().getCurrentProject();
const windows = buildAudioChordWindowsForRegion(project, audioRegion);
if (windows.length === 0) {
await showAlert('The selected audio region has no audible span to analyze.');
return;
}
setIsDetectingChords(true);
setDetectChordProgressPercent(0);
let worker: Worker | null = null;
try {
let audioBuffer = KGAudioInterface.instance().getAudioBuffer(trackId, audioRegion.getAudioFileId());
if (!audioBuffer) {
const rawBuffer = await KGAudioFileStorage.loadAudioFile(projectName, audioRegion.getAudioFileId());
const actx = Tone.getContext().rawContext as AudioContext;
audioBuffer = await actx.decodeAudioData(rawBuffer);
}
const monoPcm = new Float32Array(audioBuffer.length);
for (let channelIndex = 0; channelIndex < audioBuffer.numberOfChannels; channelIndex++) {
const channelData = audioBuffer.getChannelData(channelIndex);
for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) {
monoPcm[sampleIndex] += channelData[sampleIndex] / audioBuffer.numberOfChannels;
}
}
const request: AudioChordDetectionRequest = {
pcm: monoPcm,
sampleRate: audioBuffer.sampleRate,
clipStartOffsetSeconds: audioRegion.getClipStartOffsetSeconds(),
windows,
};
const detectedChords = await new Promise<DetectedAudioChord[]>((resolve, reject) => {
worker = new Worker(
new URL('../../workers/audioChordDetectionWorker.ts', import.meta.url),
{ type: 'module' },
);
worker.onmessage = (event: MessageEvent<AudioChordDetectionWorkerMessage>) => {
if (event.data.type === 'progress') {
setDetectChordProgressPercent(event.data.progress.percent);
return;
}
resolve(event.data.results);
};
worker.onerror = () => {
reject(new Error('Chord detection worker failed.'));
};
worker.postMessage(request, [request.pcm.buffer]);
});
const replacements = detectedChords
.filter(result => result.symbol !== 'N' && result.endBeat > result.startBeat)
.map(result => ({
startBeat: result.startBeat,
length: result.endBeat - result.startBeat,
symbol: result.symbol,
}));
const spanStartBeat = windows[0].startBeat;
const spanEndBeat = windows[windows.length - 1].endBeat;
KGCore.instance().executeCommand(
new ReplaceChordRegionsInRangeCommand(spanStartBeat, spanEndBeat, replacements),
);
refreshProjectState();
} catch (error) {
console.error('Error detecting chords:', error);
await showAlert('Failed to detect chords from this audio region.');
} finally {
if (worker) {
worker.terminate();
}
setIsDetectingChords(false);
setDetectChordProgressPercent(0);
}
}, [audioRegion, projectName, refreshProjectState, trackId]);
// Handle title click to rename the region
const handleTitleClick = () => {
// If we were just dragging, don't show the rename dialog
@@ -1315,6 +1412,8 @@ const PianoRoll: React.FC<PianoRollProps> = ({
automationType={automationType}
onAutomationToggle={handleAutomationToggle}
onAutomationTypeChange={handleAutomationTypeChange}
onDetectChords={handleDetectChords}
detectingChords={isDetectingChords}
/>
<NoteAttributeBar selectedNotes={selectedNotes} isSpectrogram={isSpectrogram} activeRegion={activeRegion} />
@@ -1351,6 +1450,8 @@ const PianoRoll: React.FC<PianoRollProps> = ({
sheetKeySignature={keySignature}
sheetInstrument={activeInstrument}
onSheetMeasureMetricsChange={handleSheetMeasureMetricsChange}
overlayMessage={isDetectingChords ? `Detecting chords… (${detectChordProgressPercent.toString().padStart(2, '0')}% completed)` : null}
overlayProgressPercent={isDetectingChords ? detectChordProgressPercent : null}
/>
<div
src/components/piano-roll/PianoRollContent.test.tsx
+14
View File
@@ -120,6 +120,20 @@ describe('PianoRollContent', () => {
expect(screen.queryByTestId('automation-lane')).not.toBeInTheDocument();
});
it('shows an overlay message when one is supplied by the parent panel', () => {
render(
<PianoRollContent
{...baseProps}
mode="spectrogram"
overlayMessage="Detecting chords…"
overlayProgressPercent={42}
/>
);
expect(screen.getByText('Detecting chords…')).toBeInTheDocument();
expect(screen.getByRole('progressbar')).toHaveAttribute('aria-valuenow', '42');
});
it('renders sheet mode without piano keys or automation lane', () => {
render(
<PianoRollContent
src/components/piano-roll/PianoRollContent.tsx
+24 -2
View File
@@ -55,6 +55,8 @@ interface PianoRollContentProps {
sheetKeySignature?: KeySignature;
sheetInstrument?: InstrumentType;
onSheetMeasureMetricsChange?: (metrics: SheetMeasureMetric[]) => void;
overlayMessage?: string | null;
overlayProgressPercent?: number | null;
}
const PianoRollContent: React.FC<PianoRollContentProps> = ({
@@ -89,6 +91,8 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
sheetKeySignature = 'C major',
sheetInstrument = 'acoustic_grand_piano',
onSheetMeasureMetricsChange,
overlayMessage = null,
overlayProgressPercent = null,
}) => {
const isSpectrogram = mode === 'spectrogram';
const showAutomationLane = automationEnabled && !isSpectrogram && !sheetMusicViewEnabled;
@@ -321,6 +325,8 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
container.scrollTop = 0;
}, [noteScrollRef, sheetMusicViewEnabled]);
const effectiveOverlayMessage = overlayMessage ?? (spectrogramLoading ? 'Computing spectrogram…' : null);
return (
<div className="piano-roll-content-outer">
<div
@@ -397,10 +403,26 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
</div>
)}
</div>
{spectrogramLoading && (
{effectiveOverlayMessage && (
<div className="spectrogram-loading-overlay">
<div className="spectrogram-loading-label">
Computing spectrogram…
{effectiveOverlayMessage}
{overlayProgressPercent !== null && (
<div className="piano-roll-progress-block">
<div
className="piano-roll-progress-track"
role="progressbar"
aria-valuemin={0}
aria-valuemax={100}
aria-valuenow={Math.max(0, Math.min(100, overlayProgressPercent))}
>
<div
className="piano-roll-progress-fill"
style={{ width: `${Math.max(0, Math.min(100, overlayProgressPercent))}%` }}
/>
</div>
</div>
)}
</div>
</div>
)}
src/components/piano-roll/PianoRollToolbar.test.tsx
+39
View File
@@ -60,6 +60,7 @@ describe('PianoRollToolbar', () => {
onChordGuideChange: vi.fn(),
zoom: 1,
onZoomChange: vi.fn(),
onDetectChords: vi.fn(),
};
it('shows automation controls in midi mode and toggles the lane', () => {
@@ -118,6 +119,44 @@ describe('PianoRollToolbar', () => {
expect(screen.queryByRole('button', { name: /Pitch Bend/i })).not.toBeInTheDocument();
});
it('shows the detect chords action in spectrogram mode and triggers it', () => {
const onDetectChords = vi.fn();
render(
<PianoRollToolbar
{...baseProps}
mode="spectrogram"
showAutomationControls={false}
onDetectChords={onDetectChords}
/>
);
fireEvent.click(screen.getByTitle('More options'));
fireEvent.click(screen.getByText('Detect chords...'));
expect(onDetectChords).toHaveBeenCalledTimes(1);
});
it('disables the detect chords action while detection is running', () => {
const onDetectChords = vi.fn();
render(
<PianoRollToolbar
{...baseProps}
mode="spectrogram"
showAutomationControls={false}
detectingChords={true}
onDetectChords={onDetectChords}
/>
);
fireEvent.click(screen.getByTitle('More options'));
const detectItem = screen.getByText('Detecting chords...');
expect(detectItem).toHaveAttribute('aria-disabled', 'true');
fireEvent.click(detectItem);
expect(onDetectChords).not.toHaveBeenCalled();
});
it('shows only the sheet controls when sheet mode is enabled', () => {
render(
<PianoRollToolbar
src/components/piano-roll/PianoRollToolbar.tsx
+47
View File
@@ -48,6 +48,8 @@ interface PianoRollToolbarProps {
automationType?: PianoRollAutomationType;
onAutomationToggle?: () => void;
onAutomationTypeChange?: (value: PianoRollAutomationType) => void;
onDetectChords?: () => void | Promise<void>;
detectingChords?: boolean;
}
const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
@@ -82,6 +84,8 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
automationType = 'pitch-bend',
onAutomationToggle,
onAutomationTypeChange,
onDetectChords,
detectingChords = false,
}) => {
const showMidiControls = mode !== 'spectrogram' && !sheetMusicViewEnabled; // midi-edit and hybrid
const showSpecControls = !sheetMusicViewEnabled && (mode === 'spectrogram' || mode === 'hybrid');
@@ -100,6 +104,20 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
return () => document.removeEventListener('mousedown', handleClickOutside);
}, [showZoomSlider]);
const [showSpecMenu, setShowSpecMenu] = React.useState(false);
const specMenuRef = React.useRef<HTMLDivElement>(null);
React.useEffect(() => {
if (!showSpecMenu) return;
const handleClickOutside = (e: MouseEvent) => {
if (specMenuRef.current && !specMenuRef.current.contains(e.target as Node)) {
setShowSpecMenu(false);
}
};
document.addEventListener('mousedown', handleClickOutside);
return () => document.removeEventListener('mousedown', handleClickOutside);
}, [showSpecMenu]);
return (
<div className="piano-roll-toolbar">
{showMidiControls && (
@@ -280,6 +298,35 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
)}
</div>
)}
{showSpecControls && (
<div className="quant-dropdown-container" ref={specMenuRef}>
<button
className="quant-button"
onClick={() => setShowSpecMenu(!showSpecMenu)}
title="More options"
>
...
</button>
{showSpecMenu && (
<div className="quant-dropdown" style={{ right: 0, left: 'auto', width: 'auto', whiteSpace: 'nowrap' }}>
<div
className={`quant-option${(!onDetectChords || detectingChords) ? ' disabled' : ''}`}
onClick={() => {
if (!onDetectChords || detectingChords) {
return;
}
setShowSpecMenu(false);
void onDetectChords();
}}
aria-disabled={!onDetectChords || detectingChords}
>
{detectingChords ? 'Detecting chords...' : 'Detect chords...'}
</div>
</div>
)}
</div>
)}
</div>
</div>
);