feat: add audio chord detection for spectrogram regions

This commit is contained in:
Xiaohan-Tian
2026-05-25 19:09:19 -07:00
parent a7158b1ee0
commit f12add935a
15 changed files with 931 additions and 8 deletions
Binary file not shown.
+35 -4
View File
@@ -235,6 +235,11 @@
left: 0; left: 0;
} }
.piano-roll-toolbar .quant-option.disabled {
color: #666;
cursor: default;
}
.piano-roll-automation-toolbar-group { .piano-roll-automation-toolbar-group {
display: flex; display: flex;
align-items: center; align-items: center;
@@ -408,11 +413,37 @@
} }
.spectrogram-loading-label { .spectrogram-loading-label {
padding: 6px 10px; min-width: 240px;
background: rgba(0, 0, 0, 0.6); padding: 10px 12px;
color: #aaa; background: rgba(0, 0, 0, 0.72);
color: #ddd;
font-size: 11px; font-size: 11px;
border-radius: 3px; border: 1px solid #3a3a3a;
border-radius: 6px;
display: flex;
flex-direction: column;
gap: 8px;
}
.piano-roll-progress-block {
display: flex;
flex-direction: column;
gap: 6px;
}
.piano-roll-progress-track {
width: 100%;
height: 8px;
background-color: #1d1d1d;
border: 1px solid #3a3a3a;
border-radius: 999px;
overflow: hidden;
}
.piano-roll-progress-fill {
height: 100%;
background: linear-gradient(90deg, #5a9fd4 0%, #7cc2f1 100%);
transition: width 0.15s linear;
} }
.piano-grid-header { .piano-grid-header {
+103 -2
View File
@@ -1,5 +1,6 @@
import React, { useRef, useEffect, useState, useCallback, useLayoutEffect, useMemo } from 'react'; import React, { useRef, useEffect, useState, useCallback, useLayoutEffect, useMemo } from 'react';
import './PianoRoll.css'; import './PianoRoll.css';
import * as Tone from 'tone';
import { useProjectStore } from '../../stores/projectStore'; import { useProjectStore } from '../../stores/projectStore';
import { FaGripLines } from 'react-icons/fa'; import { FaGripLines } from 'react-icons/fa';
import { KGMidiRegion } from '../../core/region/KGMidiRegion'; import { KGMidiRegion } from '../../core/region/KGMidiRegion';
@@ -15,13 +16,21 @@ import { KGMidiTrack, type InstrumentType } from '../../core/track/KGMidiTrack';
import { KGPianoRollState } from '../../core/state/KGPianoRollState'; import { KGPianoRollState } from '../../core/state/KGPianoRollState';
import { ConfigManager } from '../../core/config/ConfigManager'; import { ConfigManager } from '../../core/config/ConfigManager';
import { beatsToBar } from '../../util/midiUtil'; import { beatsToBar } from '../../util/midiUtil';
import { UpdateRegionCommand } from '../../core/commands'; import { ReplaceChordRegionsInRangeCommand, UpdateRegionCommand } from '../../core/commands';
import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage';
import { getSuitableChords, noteNameToPitchClass } from '../../util/scaleUtil'; import { getSuitableChords, noteNameToPitchClass } from '../../util/scaleUtil';
import { showAlert } from '../../util/dialogUtil'; import { showAlert } from '../../util/dialogUtil';
import { import {
normalizeSpectrogramHeightResolution, normalizeSpectrogramHeightResolution,
type SpectrogramHeightResolution, type SpectrogramHeightResolution,
} from '../../util/spectrogramUtil'; } from '../../util/spectrogramUtil';
import {
buildAudioChordWindowsForRegion,
type AudioChordDetectionRequest,
type DetectedAudioChord,
} from '../../util/audioChordDetection';
import type { AudioChordDetectionWorkerMessage } from '../../workers/audioChordDetectionWorker';
import type { PianoRollAutomationType } from './pianoRollAutomation'; import type { PianoRollAutomationType } from './pianoRollAutomation';
import type { SheetMeasureMetric } from './sheetNotationTypes'; import type { SheetMeasureMetric } from './sheetNotationTypes';
import { getSheetPlayheadPixel, getSheetQuantizationOptions, parseSheetQuantization } from './sheetNotation'; import { getSheetPlayheadPixel, getSheetQuantizationOptions, parseSheetQuantization } from './sheetNotation';
@@ -59,7 +68,7 @@ const PianoRoll: React.FC<PianoRollProps> = ({
}) => { }) => {
const isSpectrogram = mode === 'spectrogram'; const isSpectrogram = mode === 'spectrogram';
const isHybrid = mode === 'hybrid'; const isHybrid = mode === 'hybrid';
const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showKGOnePanel, showEventListPanel, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled, bpm, pianoRollScrollRequest, selectedNoteIds, automationRedrawVersion } = useProjectStore(); const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showKGOnePanel, showEventListPanel, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled, bpm, pianoRollScrollRequest, selectedNoteIds, automationRedrawVersion, refreshProjectState } = useProjectStore();
// Tool state for piano roll // Tool state for piano roll
const [activeTool, setActiveTool] = useState<'pointer' | 'pencil'>('pointer'); const [activeTool, setActiveTool] = useState<'pointer' | 'pencil'>('pointer');
@@ -69,6 +78,8 @@ const PianoRoll: React.FC<PianoRollProps> = ({
const [spectrogramPower, setSpectrogramPower] = useState<number>(0.5); const [spectrogramPower, setSpectrogramPower] = useState<number>(0.5);
const [spectrogramHeightResolution, setSpectrogramHeightResolution] = const [spectrogramHeightResolution, setSpectrogramHeightResolution] =
useState<SpectrogramHeightResolution>(3); useState<SpectrogramHeightResolution>(3);
const [isDetectingChords, setIsDetectingChords] = useState(false);
const [detectChordProgressPercent, setDetectChordProgressPercent] = useState(0);
// Piano roll zoom (1x8x); updates --region-grid-beat-width CSS variable // Piano roll zoom (1x8x); updates --region-grid-beat-width CSS variable
const [pianoRollZoom, setPianoRollZoom] = useState<number>(() => KGPianoRollState.instance().getPianoRollZoom()); const [pianoRollZoom, setPianoRollZoom] = useState<number>(() => KGPianoRollState.instance().getPianoRollZoom());
@@ -436,6 +447,92 @@ const PianoRoll: React.FC<PianoRollProps> = ({
} }
}; };
const handleDetectChords = useCallback(async () => {
if (!audioRegion || !projectName || !trackId) {
await showAlert('Open an audio region in spectrogram mode before detecting chords.');
return;
}
const project = KGCore.instance().getCurrentProject();
const windows = buildAudioChordWindowsForRegion(project, audioRegion);
if (windows.length === 0) {
await showAlert('The selected audio region has no audible span to analyze.');
return;
}
setIsDetectingChords(true);
setDetectChordProgressPercent(0);
let worker: Worker | null = null;
try {
let audioBuffer = KGAudioInterface.instance().getAudioBuffer(trackId, audioRegion.getAudioFileId());
if (!audioBuffer) {
const rawBuffer = await KGAudioFileStorage.loadAudioFile(projectName, audioRegion.getAudioFileId());
const actx = Tone.getContext().rawContext as AudioContext;
audioBuffer = await actx.decodeAudioData(rawBuffer);
}
const monoPcm = new Float32Array(audioBuffer.length);
for (let channelIndex = 0; channelIndex < audioBuffer.numberOfChannels; channelIndex++) {
const channelData = audioBuffer.getChannelData(channelIndex);
for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) {
monoPcm[sampleIndex] += channelData[sampleIndex] / audioBuffer.numberOfChannels;
}
}
const request: AudioChordDetectionRequest = {
pcm: monoPcm,
sampleRate: audioBuffer.sampleRate,
clipStartOffsetSeconds: audioRegion.getClipStartOffsetSeconds(),
windows,
};
const detectedChords = await new Promise<DetectedAudioChord[]>((resolve, reject) => {
worker = new Worker(
new URL('../../workers/audioChordDetectionWorker.ts', import.meta.url),
{ type: 'module' },
);
worker.onmessage = (event: MessageEvent<AudioChordDetectionWorkerMessage>) => {
if (event.data.type === 'progress') {
setDetectChordProgressPercent(event.data.progress.percent);
return;
}
resolve(event.data.results);
};
worker.onerror = () => {
reject(new Error('Chord detection worker failed.'));
};
worker.postMessage(request, [request.pcm.buffer]);
});
const replacements = detectedChords
.filter(result => result.symbol !== 'N' && result.endBeat > result.startBeat)
.map(result => ({
startBeat: result.startBeat,
length: result.endBeat - result.startBeat,
symbol: result.symbol,
}));
const spanStartBeat = windows[0].startBeat;
const spanEndBeat = windows[windows.length - 1].endBeat;
KGCore.instance().executeCommand(
new ReplaceChordRegionsInRangeCommand(spanStartBeat, spanEndBeat, replacements),
);
refreshProjectState();
} catch (error) {
console.error('Error detecting chords:', error);
await showAlert('Failed to detect chords from this audio region.');
} finally {
if (worker) {
worker.terminate();
}
setIsDetectingChords(false);
setDetectChordProgressPercent(0);
}
}, [audioRegion, projectName, refreshProjectState, trackId]);
// Handle title click to rename the region // Handle title click to rename the region
const handleTitleClick = () => { const handleTitleClick = () => {
// If we were just dragging, don't show the rename dialog // If we were just dragging, don't show the rename dialog
@@ -1315,6 +1412,8 @@ const PianoRoll: React.FC<PianoRollProps> = ({
automationType={automationType} automationType={automationType}
onAutomationToggle={handleAutomationToggle} onAutomationToggle={handleAutomationToggle}
onAutomationTypeChange={handleAutomationTypeChange} onAutomationTypeChange={handleAutomationTypeChange}
onDetectChords={handleDetectChords}
detectingChords={isDetectingChords}
/> />
<NoteAttributeBar selectedNotes={selectedNotes} isSpectrogram={isSpectrogram} activeRegion={activeRegion} /> <NoteAttributeBar selectedNotes={selectedNotes} isSpectrogram={isSpectrogram} activeRegion={activeRegion} />
@@ -1351,6 +1450,8 @@ const PianoRoll: React.FC<PianoRollProps> = ({
sheetKeySignature={keySignature} sheetKeySignature={keySignature}
sheetInstrument={activeInstrument} sheetInstrument={activeInstrument}
onSheetMeasureMetricsChange={handleSheetMeasureMetricsChange} onSheetMeasureMetricsChange={handleSheetMeasureMetricsChange}
overlayMessage={isDetectingChords ? `Detecting chords… (${detectChordProgressPercent.toString().padStart(2, '0')}% completed)` : null}
overlayProgressPercent={isDetectingChords ? detectChordProgressPercent : null}
/> />
<div <div
@@ -120,6 +120,20 @@ describe('PianoRollContent', () => {
expect(screen.queryByTestId('automation-lane')).not.toBeInTheDocument(); expect(screen.queryByTestId('automation-lane')).not.toBeInTheDocument();
}); });
it('shows an overlay message when one is supplied by the parent panel', () => {
render(
<PianoRollContent
{...baseProps}
mode="spectrogram"
overlayMessage="Detecting chords…"
overlayProgressPercent={42}
/>
);
expect(screen.getByText('Detecting chords…')).toBeInTheDocument();
expect(screen.getByRole('progressbar')).toHaveAttribute('aria-valuenow', '42');
});
it('renders sheet mode without piano keys or automation lane', () => { it('renders sheet mode without piano keys or automation lane', () => {
render( render(
<PianoRollContent <PianoRollContent
+24 -2
View File
@@ -55,6 +55,8 @@ interface PianoRollContentProps {
sheetKeySignature?: KeySignature; sheetKeySignature?: KeySignature;
sheetInstrument?: InstrumentType; sheetInstrument?: InstrumentType;
onSheetMeasureMetricsChange?: (metrics: SheetMeasureMetric[]) => void; onSheetMeasureMetricsChange?: (metrics: SheetMeasureMetric[]) => void;
overlayMessage?: string | null;
overlayProgressPercent?: number | null;
} }
const PianoRollContent: React.FC<PianoRollContentProps> = ({ const PianoRollContent: React.FC<PianoRollContentProps> = ({
@@ -89,6 +91,8 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
sheetKeySignature = 'C major', sheetKeySignature = 'C major',
sheetInstrument = 'acoustic_grand_piano', sheetInstrument = 'acoustic_grand_piano',
onSheetMeasureMetricsChange, onSheetMeasureMetricsChange,
overlayMessage = null,
overlayProgressPercent = null,
}) => { }) => {
const isSpectrogram = mode === 'spectrogram'; const isSpectrogram = mode === 'spectrogram';
const showAutomationLane = automationEnabled && !isSpectrogram && !sheetMusicViewEnabled; const showAutomationLane = automationEnabled && !isSpectrogram && !sheetMusicViewEnabled;
@@ -321,6 +325,8 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
container.scrollTop = 0; container.scrollTop = 0;
}, [noteScrollRef, sheetMusicViewEnabled]); }, [noteScrollRef, sheetMusicViewEnabled]);
const effectiveOverlayMessage = overlayMessage ?? (spectrogramLoading ? 'Computing spectrogram…' : null);
return ( return (
<div className="piano-roll-content-outer"> <div className="piano-roll-content-outer">
<div <div
@@ -397,10 +403,26 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
</div> </div>
)} )}
</div> </div>
{spectrogramLoading && ( {effectiveOverlayMessage && (
<div className="spectrogram-loading-overlay"> <div className="spectrogram-loading-overlay">
<div className="spectrogram-loading-label"> <div className="spectrogram-loading-label">
Computing spectrogram {effectiveOverlayMessage}
{overlayProgressPercent !== null && (
<div className="piano-roll-progress-block">
<div
className="piano-roll-progress-track"
role="progressbar"
aria-valuemin={0}
aria-valuemax={100}
aria-valuenow={Math.max(0, Math.min(100, overlayProgressPercent))}
>
<div
className="piano-roll-progress-fill"
style={{ width: `${Math.max(0, Math.min(100, overlayProgressPercent))}%` }}
/>
</div>
</div>
)}
</div> </div>
</div> </div>
)} )}
@@ -60,6 +60,7 @@ describe('PianoRollToolbar', () => {
onChordGuideChange: vi.fn(), onChordGuideChange: vi.fn(),
zoom: 1, zoom: 1,
onZoomChange: vi.fn(), onZoomChange: vi.fn(),
onDetectChords: vi.fn(),
}; };
it('shows automation controls in midi mode and toggles the lane', () => { it('shows automation controls in midi mode and toggles the lane', () => {
@@ -118,6 +119,44 @@ describe('PianoRollToolbar', () => {
expect(screen.queryByRole('button', { name: /Pitch Bend/i })).not.toBeInTheDocument(); expect(screen.queryByRole('button', { name: /Pitch Bend/i })).not.toBeInTheDocument();
}); });
it('shows the detect chords action in spectrogram mode and triggers it', () => {
const onDetectChords = vi.fn();
render(
<PianoRollToolbar
{...baseProps}
mode="spectrogram"
showAutomationControls={false}
onDetectChords={onDetectChords}
/>
);
fireEvent.click(screen.getByTitle('More options'));
fireEvent.click(screen.getByText('Detect chords...'));
expect(onDetectChords).toHaveBeenCalledTimes(1);
});
it('disables the detect chords action while detection is running', () => {
const onDetectChords = vi.fn();
render(
<PianoRollToolbar
{...baseProps}
mode="spectrogram"
showAutomationControls={false}
detectingChords={true}
onDetectChords={onDetectChords}
/>
);
fireEvent.click(screen.getByTitle('More options'));
const detectItem = screen.getByText('Detecting chords...');
expect(detectItem).toHaveAttribute('aria-disabled', 'true');
fireEvent.click(detectItem);
expect(onDetectChords).not.toHaveBeenCalled();
});
it('shows only the sheet controls when sheet mode is enabled', () => { it('shows only the sheet controls when sheet mode is enabled', () => {
render( render(
<PianoRollToolbar <PianoRollToolbar
@@ -48,6 +48,8 @@ interface PianoRollToolbarProps {
automationType?: PianoRollAutomationType; automationType?: PianoRollAutomationType;
onAutomationToggle?: () => void; onAutomationToggle?: () => void;
onAutomationTypeChange?: (value: PianoRollAutomationType) => void; onAutomationTypeChange?: (value: PianoRollAutomationType) => void;
onDetectChords?: () => void | Promise<void>;
detectingChords?: boolean;
} }
const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
@@ -82,6 +84,8 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
automationType = 'pitch-bend', automationType = 'pitch-bend',
onAutomationToggle, onAutomationToggle,
onAutomationTypeChange, onAutomationTypeChange,
onDetectChords,
detectingChords = false,
}) => { }) => {
const showMidiControls = mode !== 'spectrogram' && !sheetMusicViewEnabled; // midi-edit and hybrid const showMidiControls = mode !== 'spectrogram' && !sheetMusicViewEnabled; // midi-edit and hybrid
const showSpecControls = !sheetMusicViewEnabled && (mode === 'spectrogram' || mode === 'hybrid'); const showSpecControls = !sheetMusicViewEnabled && (mode === 'spectrogram' || mode === 'hybrid');
@@ -100,6 +104,20 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
return () => document.removeEventListener('mousedown', handleClickOutside); return () => document.removeEventListener('mousedown', handleClickOutside);
}, [showZoomSlider]); }, [showZoomSlider]);
const [showSpecMenu, setShowSpecMenu] = React.useState(false);
const specMenuRef = React.useRef<HTMLDivElement>(null);
React.useEffect(() => {
if (!showSpecMenu) return;
const handleClickOutside = (e: MouseEvent) => {
if (specMenuRef.current && !specMenuRef.current.contains(e.target as Node)) {
setShowSpecMenu(false);
}
};
document.addEventListener('mousedown', handleClickOutside);
return () => document.removeEventListener('mousedown', handleClickOutside);
}, [showSpecMenu]);
return ( return (
<div className="piano-roll-toolbar"> <div className="piano-roll-toolbar">
{showMidiControls && ( {showMidiControls && (
@@ -280,6 +298,35 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
)} )}
</div> </div>
)} )}
{showSpecControls && (
<div className="quant-dropdown-container" ref={specMenuRef}>
<button
className="quant-button"
onClick={() => setShowSpecMenu(!showSpecMenu)}
title="More options"
>
...
</button>
{showSpecMenu && (
<div className="quant-dropdown" style={{ right: 0, left: 'auto', width: 'auto', whiteSpace: 'nowrap' }}>
<div
className={`quant-option${(!onDetectChords || detectingChords) ? ' disabled' : ''}`}
onClick={() => {
if (!onDetectChords || detectingChords) {
return;
}
setShowSpecMenu(false);
void onDetectChords();
}}
aria-disabled={!onDetectChords || detectingChords}
>
{detectingChords ? 'Detecting chords...' : 'Detect chords...'}
</div>
</div>
)}
</div>
)}
</div> </div>
</div> </div>
); );
@@ -6,6 +6,7 @@ import { KGChordRegion } from '../../region/KGChordRegion';
import { CreateChordRegionCommand } from './CreateChordRegionCommand'; import { CreateChordRegionCommand } from './CreateChordRegionCommand';
import { InsertChordRegionAtBeatCommand } from './InsertChordRegionAtBeatCommand'; import { InsertChordRegionAtBeatCommand } from './InsertChordRegionAtBeatCommand';
import { MoveGlobalRegionCommand } from './MoveGlobalRegionCommand'; import { MoveGlobalRegionCommand } from './MoveGlobalRegionCommand';
import { ReplaceChordRegionsInRangeCommand } from './ReplaceChordRegionsInRangeCommand';
import { ResizeGlobalRegionCommand } from './ResizeGlobalRegionCommand'; import { ResizeGlobalRegionCommand } from './ResizeGlobalRegionCommand';
import { UpdateChordRegionCommand } from './UpdateChordRegionCommand'; import { UpdateChordRegionCommand } from './UpdateChordRegionCommand';
@@ -104,4 +105,46 @@ describe('global chord region commands', () => {
expect(chordTrack.getRegions()).toHaveLength(1); expect(chordTrack.getRegions()).toHaveLength(1);
expect(region.getLength()).toBe(8); expect(region.getLength()).toBe(8);
}); });
it('replaces only the requested chord span and restores the original layout on undo', () => {
const chordTrack = getChordTrack();
chordTrack.setRegions([
new KGChordRegion('left', chordTrack.getId(), chordTrack.getTrackIndex(), 'C', 0, 4),
new KGChordRegion('middle', chordTrack.getId(), chordTrack.getTrackIndex(), 'Am', 4, 4),
new KGChordRegion('right', chordTrack.getId(), chordTrack.getTrackIndex(), 'F', 8, 4),
]);
const command = new ReplaceChordRegionsInRangeCommand(2, 10, [
{ startBeat: 2, length: 2, symbol: 'Dm' },
{ startBeat: 4, length: 4, symbol: 'E' },
{ startBeat: 8, length: 2, symbol: 'Am' },
]);
command.execute();
const replacedRegions = getChordTrack().getRegions() as KGChordRegion[];
expect(replacedRegions.map(region => ({
symbol: region.getSymbol(),
start: region.getStartFromBeat(),
length: region.getLength(),
}))).toEqual([
{ symbol: 'C', start: 0, length: 2 },
{ symbol: 'Dm', start: 2, length: 2 },
{ symbol: 'E', start: 4, length: 4 },
{ symbol: 'Am', start: 8, length: 2 },
{ symbol: 'F', start: 10, length: 2 },
]);
command.undo();
const restoredRegions = getChordTrack().getRegions() as KGChordRegion[];
expect(restoredRegions.map(region => ({
symbol: region.getSymbol(),
start: region.getStartFromBeat(),
length: region.getLength(),
}))).toEqual([
{ symbol: 'C', start: 0, length: 4 },
{ symbol: 'Am', start: 4, length: 4 },
{ symbol: 'F', start: 8, length: 4 },
]);
});
}); });
@@ -0,0 +1,132 @@
import { KGCommand } from '../KGCommand';
import { KGCore } from '../../KGCore';
import { GlobalTrackType } from '../../global-track';
import { KGChordRegion } from '../../region/KGChordRegion';
import { findGlobalTrackByType } from '../../../util/globalTrackUtil';
import { generateUniqueId } from '../../../util/miscUtil';
export interface ChordRegionReplacementData {
startBeat: number;
length: number;
symbol: string;
}
function cloneChordRegion(region: KGChordRegion): KGChordRegion {
return new KGChordRegion(
region.getId(),
region.getTrackId(),
region.getTrackIndex(),
region.getSymbol(),
region.getStartFromBeat(),
region.getLength(),
);
}
function cloneChordRegions(regions: KGChordRegion[]): KGChordRegion[] {
return regions.map(cloneChordRegion);
}
export class ReplaceChordRegionsInRangeCommand extends KGCommand {
private readonly rangeStartBeat: number;
private readonly rangeEndBeat: number;
private readonly replacements: ChordRegionReplacementData[];
private originalRegions: KGChordRegion[] | null = null;
private nextRegions: KGChordRegion[] | null = null;
constructor(rangeStartBeat: number, rangeEndBeat: number, replacements: ChordRegionReplacementData[]) {
super();
this.rangeStartBeat = Math.max(0, rangeStartBeat);
this.rangeEndBeat = Math.max(this.rangeStartBeat, rangeEndBeat);
this.replacements = replacements.map(replacement => ({
startBeat: replacement.startBeat,
length: replacement.length,
symbol: replacement.symbol,
}));
}
execute(): void {
const project = KGCore.instance().getCurrentProject();
const chordTrack = findGlobalTrackByType(project, GlobalTrackType.Chord);
if (!chordTrack) {
throw new Error('Chord global track not found');
}
if (this.nextRegions) {
chordTrack.setRegions(cloneChordRegions(this.nextRegions));
return;
}
const currentRegions = chordTrack.getRegions()
.filter((region): region is KGChordRegion => region instanceof KGChordRegion)
.sort((left, right) => left.getStartFromBeat() - right.getStartFromBeat());
this.originalRegions = cloneChordRegions(currentRegions);
const preservedRegions: KGChordRegion[] = [];
for (const region of currentRegions) {
const regionStart = region.getStartFromBeat();
const regionEnd = regionStart + region.getLength();
if (regionEnd <= this.rangeStartBeat || regionStart >= this.rangeEndBeat) {
preservedRegions.push(cloneChordRegion(region));
continue;
}
if (regionStart < this.rangeStartBeat) {
preservedRegions.push(new KGChordRegion(
region.getId(),
region.getTrackId(),
region.getTrackIndex(),
region.getSymbol(),
regionStart,
this.rangeStartBeat - regionStart,
));
}
if (regionEnd > this.rangeEndBeat) {
preservedRegions.push(new KGChordRegion(
generateUniqueId('KGChordRegion'),
region.getTrackId(),
region.getTrackIndex(),
region.getSymbol(),
this.rangeEndBeat,
regionEnd - this.rangeEndBeat,
));
}
}
const replacementRegions = this.replacements
.filter(replacement => replacement.length > 0 && replacement.symbol.trim() !== '')
.map(replacement => new KGChordRegion(
generateUniqueId('KGChordRegion'),
chordTrack.getId(),
chordTrack.getTrackIndex(),
replacement.symbol,
replacement.startBeat,
replacement.length,
));
this.nextRegions = [...preservedRegions, ...replacementRegions]
.sort((left, right) => left.getStartFromBeat() - right.getStartFromBeat());
chordTrack.setRegions(cloneChordRegions(this.nextRegions));
}
undo(): void {
if (!this.originalRegions) {
throw new Error('Cannot undo chord replacement without original regions');
}
const project = KGCore.instance().getCurrentProject();
const chordTrack = findGlobalTrackByType(project, GlobalTrackType.Chord);
if (!chordTrack) {
throw new Error('Chord global track not found during undo');
}
chordTrack.setRegions(cloneChordRegions(this.originalRegions));
}
getDescription(): string {
return 'Replace chord regions in range';
}
}
+4
View File
@@ -39,6 +39,10 @@ export { MergeMidiRegionsCommand } from './region/MergeMidiRegionsCommand';
export { CreateGlobalMarkerRegionCommand } from './global-region/CreateGlobalMarkerRegionCommand'; export { CreateGlobalMarkerRegionCommand } from './global-region/CreateGlobalMarkerRegionCommand';
export { CreateChordRegionCommand } from './global-region/CreateChordRegionCommand'; export { CreateChordRegionCommand } from './global-region/CreateChordRegionCommand';
export { InsertChordRegionAtBeatCommand } from './global-region/InsertChordRegionAtBeatCommand'; export { InsertChordRegionAtBeatCommand } from './global-region/InsertChordRegionAtBeatCommand';
export {
ReplaceChordRegionsInRangeCommand,
type ChordRegionReplacementData,
} from './global-region/ReplaceChordRegionsInRangeCommand';
export { CreateKeySignatureRegionCommand } from './global-region/CreateKeySignatureRegionCommand'; export { CreateKeySignatureRegionCommand } from './global-region/CreateKeySignatureRegionCommand';
export { CreateTempoRegionCommand } from './global-region/CreateTempoRegionCommand'; export { CreateTempoRegionCommand } from './global-region/CreateTempoRegionCommand';
export { MoveGlobalRegionCommand } from './global-region/MoveGlobalRegionCommand'; export { MoveGlobalRegionCommand } from './global-region/MoveGlobalRegionCommand';
@@ -0,0 +1,80 @@
import { execFileSync, spawnSync } from 'node:child_process';
import path from 'node:path';
import { describe, expect, it } from 'vitest';
import { detectChordsFromAudio } from '../../util/audioChordDetectionCore';
const FIXTURE_PATH = path.resolve(process.cwd(), 'public/test-data/chord-progression-01.mp3');
const BAR_DURATION_SECONDS = 2;
const ffmpegAvailable = spawnSync('ffmpeg', ['-version'], { stdio: 'ignore' }).status === 0;
const runIfFfmpeg = ffmpegAvailable ? it : it.skip;
function decodeMp3ToMonoPcm(path: string): { sampleRate: number; pcm: Float32Array } {
const wav = execFileSync(
'ffmpeg',
['-v', 'error', '-i', path, '-ac', '1', '-f', 'wav', 'pipe:1'],
{ maxBuffer: 64 * 1024 * 1024 },
);
const readString = (offset: number, length: number) => wav.toString('ascii', offset, offset + length);
if (readString(0, 4) !== 'RIFF' || readString(8, 4) !== 'WAVE') {
throw new Error('ffmpeg did not return a RIFF/WAVE stream');
}
let offset = 12;
let sampleRate = 44100;
let pcmData = Buffer.alloc(0);
while (offset + 8 <= wav.length) {
const chunkId = readString(offset, 4);
const chunkSize = wav.readUInt32LE(offset + 4);
const chunkStart = offset + 8;
if (chunkId === 'fmt ') {
sampleRate = wav.readUInt32LE(chunkStart + 4);
} else if (chunkId === 'data') {
pcmData = wav.subarray(chunkStart, chunkStart + chunkSize);
break;
}
offset = chunkStart + chunkSize + (chunkSize % 2);
}
const pcm = new Float32Array(pcmData.length / 2);
for (let sampleIndex = 0; sampleIndex < pcm.length; sampleIndex++) {
pcm[sampleIndex] = pcmData.readInt16LE(sampleIndex * 2) / 32768;
}
return { sampleRate, pcm };
}
describe('audio chord detection fixture', () => {
runIfFfmpeg('detects the expected bar-locked progression from the mp3 fixture', () => {
const { sampleRate, pcm } = decodeMp3ToMonoPcm(FIXTURE_PATH);
const windows = Array.from({ length: 9 }, (_, barIndex) => ({
barIndex,
startBeat: barIndex * 4,
endBeat: (barIndex + 1) * 4,
startSeconds: barIndex * BAR_DURATION_SECONDS,
endSeconds: (barIndex + 1) * BAR_DURATION_SECONDS,
}));
const results = detectChordsFromAudio({
pcm,
sampleRate,
clipStartOffsetSeconds: 0,
windows,
});
expect(results.map(result => result.symbol)).toEqual([
'Am',
'F',
'Dm',
'E',
'Am',
'C',
'Dm',
'E',
'N',
]);
});
});
+75
View File
@@ -0,0 +1,75 @@
import { describe, expect, it } from 'vitest';
import { detectChordsFromAudio, type AudioChordDetectionRequest } from './audioChordDetectionCore';
const SAMPLE_RATE = 44100;
function createSineChordPcm(frequencies: number[], durationSeconds: number): Float32Array {
const sampleCount = Math.floor(durationSeconds * SAMPLE_RATE);
const pcm = new Float32Array(sampleCount);
for (let sampleIndex = 0; sampleIndex < sampleCount; sampleIndex++) {
const time = sampleIndex / SAMPLE_RATE;
let sample = 0;
for (const frequency of frequencies) {
sample += Math.sin(2 * Math.PI * frequency * time);
}
pcm[sampleIndex] = (sample / Math.max(1, frequencies.length)) * 0.35;
}
return pcm;
}
function createRequest(windows: AudioChordDetectionRequest['windows'], pcm: Float32Array): AudioChordDetectionRequest {
return {
pcm,
sampleRate: SAMPLE_RATE,
clipStartOffsetSeconds: 0,
windows,
};
}
describe('audio chord detection', () => {
it('detects a major triad from synthetic audio', () => {
const pcm = createSineChordPcm([261.63, 329.63, 392.0], 2);
const [result] = detectChordsFromAudio(createRequest([
{ barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 },
], pcm));
expect(result.symbol).toBe('C');
expect(result.confidence).toBeGreaterThan(0);
});
it('detects a minor triad from synthetic audio', () => {
const pcm = createSineChordPcm([220.0, 261.63, 329.63], 2);
const [result] = detectChordsFromAudio(createRequest([
{ barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 },
], pcm));
expect(result.symbol).toBe('Am');
});
it('marks silent analysis windows as no chord', () => {
const pcm = new Float32Array(SAMPLE_RATE * 2);
const [result] = detectChordsFromAudio(createRequest([
{ barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 },
], pcm));
expect(result.symbol).toBe('N');
expect(result.confidence).toBe(0);
});
it('keeps neighboring synthetic bars stable', () => {
const barA = createSineChordPcm([220.0, 261.63, 329.63], 2);
const barB = createSineChordPcm([220.0, 261.63, 329.63], 2);
const pcm = new Float32Array(barA.length + barB.length);
pcm.set(barA, 0);
pcm.set(barB, barA.length);
const results = detectChordsFromAudio(createRequest([
{ barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 },
{ barIndex: 1, startBeat: 4, endBeat: 8, startSeconds: 2, endSeconds: 4 },
], pcm));
expect(results.map(result => result.symbol)).toEqual(['Am', 'Am']);
});
});
+55
View File
@@ -0,0 +1,55 @@
import { KGProject } from '../core/KGProject';
import { KGAudioRegion } from '../core/region/KGAudioRegion';
import { beatRangeToSeconds, getAudioRegionDisplayLengthBeats } from './globalTrackUtil';
export {
detectChordsFromAudio,
type AudioChordDetectionRequest,
type AudioChordWindow,
type DetectedAudioChord,
} from './audioChordDetectionCore';
import type { AudioChordWindow } from './audioChordDetectionCore';
export function buildAudioChordWindowsForRegion(
project: KGProject,
audioRegion: KGAudioRegion,
): AudioChordWindow[] {
const regionStartBeat = audioRegion.getStartFromBeat();
const visibleLengthBeats = getAudioRegionDisplayLengthBeats(project, audioRegion);
if (visibleLengthBeats <= 0) {
return [];
}
const regionEndBeat = regionStartBeat + visibleLengthBeats;
const beatsPerBar = project.getTimeSignature().numerator;
const startBarIndex = Math.floor(regionStartBeat / beatsPerBar);
const lastBeatExclusive = regionEndBeat - 1e-9;
const endBarIndexExclusive = Math.max(
startBarIndex + 1,
Math.ceil(Math.max(regionStartBeat, lastBeatExclusive) / beatsPerBar),
);
const windows: AudioChordWindow[] = [];
for (let barIndex = startBarIndex; barIndex < endBarIndexExclusive; barIndex++) {
const barStartBeat = barIndex * beatsPerBar;
const barEndBeat = barStartBeat + beatsPerBar;
const overlapStartBeat = Math.max(regionStartBeat, barStartBeat);
const overlapEndBeat = Math.min(regionEndBeat, barEndBeat);
if (overlapEndBeat <= overlapStartBeat) {
continue;
}
const startSeconds = audioRegion.getClipStartOffsetSeconds() + beatRangeToSeconds(project, regionStartBeat, overlapStartBeat);
const endSeconds = audioRegion.getClipStartOffsetSeconds() + beatRangeToSeconds(project, regionStartBeat, overlapEndBeat);
windows.push({
barIndex,
startBeat: overlapStartBeat,
endBeat: overlapEndBeat,
startSeconds,
endSeconds,
});
}
return windows;
}
+256
View File
@@ -0,0 +1,256 @@
import FFT from 'fft.js';
const FFT_SIZE = 4096;
const HOP_SIZE = 512;
const MIN_ANALYSIS_FREQUENCY = 55;
const MAX_ANALYSIS_FREQUENCY = 1800;
const ABSOLUTE_SILENCE_RMS = 0.0025;
const RELATIVE_SILENCE_RATIO = 0.2;
const MIN_WINDOW_DURATION_SECONDS = 0.08;
const ROOT_NAMES = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B'] as const;
export interface AudioChordWindow {
barIndex: number;
startBeat: number;
endBeat: number;
startSeconds: number;
endSeconds: number;
}
export interface AudioChordDetectionRequest {
pcm: Float32Array;
sampleRate: number;
clipStartOffsetSeconds: number;
windows: AudioChordWindow[];
}
export interface DetectedAudioChord {
barIndex: number;
startBeat: number;
endBeat: number;
symbol: string;
confidence: number;
rms: number;
}
export interface AudioChordDetectionProgress {
completedWindows: number;
totalWindows: number;
percent: number;
}
interface ScoredChord {
symbol: string;
score: number;
}
const HANN_WINDOW = (() => {
const window = new Float32Array(FFT_SIZE);
for (let i = 0; i < FFT_SIZE; i++) {
window[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / (FFT_SIZE - 1)));
}
return window;
})();
function clamp(value: number, min: number, max: number): number {
return Math.max(min, Math.min(max, value));
}
function cloneWindow(window: AudioChordWindow): AudioChordWindow {
return { ...window };
}
function buildTriadCandidates(chroma: Float64Array): { best: ScoredChord; second: ScoredChord } {
let best: ScoredChord = { symbol: 'N', score: Number.NEGATIVE_INFINITY };
let second: ScoredChord = { symbol: 'N', score: Number.NEGATIVE_INFINITY };
for (let root = 0; root < ROOT_NAMES.length; root++) {
const rootEnergy = chroma[root];
const minorThird = chroma[(root + 3) % 12];
const majorThird = chroma[(root + 4) % 12];
const fifth = chroma[(root + 7) % 12];
const outsideEnergy = Math.max(0, 1 - (rootEnergy + minorThird + majorThird + fifth));
const majorScore = (rootEnergy * 1.2) + (majorThird * 1.0) + (fifth * 0.8) - (outsideEnergy * 0.35) - (minorThird * 0.5);
const minorScore = (rootEnergy * 1.2) + (minorThird * 1.0) + (fifth * 0.8) - (outsideEnergy * 0.35) - (majorThird * 0.5);
const candidates: ScoredChord[] = [
{ symbol: ROOT_NAMES[root], score: majorScore },
{ symbol: `${ROOT_NAMES[root]}m`, score: minorScore },
];
for (const candidate of candidates) {
if (candidate.score > best.score) {
second = best;
best = candidate;
} else if (candidate.score > second.score) {
second = candidate;
}
}
}
return { best, second };
}
function analyzeChordWindow(
pcm: Float32Array,
sampleRate: number,
startSeconds: number,
endSeconds: number,
): { symbol: string; confidence: number; rms: number } {
const startSample = Math.max(0, Math.floor(startSeconds * sampleRate));
const endSample = Math.min(pcm.length, Math.ceil(endSeconds * sampleRate));
const sampleCount = Math.max(0, endSample - startSample);
if (sampleCount === 0 || (endSeconds - startSeconds) < MIN_WINDOW_DURATION_SECONDS) {
return { symbol: 'N', confidence: 0, rms: 0 };
}
const input = new Float32Array(FFT_SIZE);
const fft = new FFT(FFT_SIZE);
const output = fft.createComplexArray() as number[];
const chroma = new Float64Array(12);
let totalEnergy = 0;
for (let i = startSample; i < endSample; i++) {
const value = pcm[i];
totalEnergy += value * value;
}
const rms = Math.sqrt(totalEnergy / sampleCount);
const totalHops = Math.max(1, Math.floor(Math.max(0, sampleCount - FFT_SIZE) / HOP_SIZE) + 1);
for (let hop = 0; hop < totalHops; hop++) {
input.fill(0);
const frameOffset = startSample + (hop * HOP_SIZE);
const available = Math.max(0, Math.min(FFT_SIZE, endSample - frameOffset));
let frameEnergy = 0;
for (let i = 0; i < available; i++) {
const weighted = pcm[frameOffset + i] * HANN_WINDOW[i];
input[i] = weighted;
frameEnergy += weighted * weighted;
}
if (frameEnergy < 1e-7) {
continue;
}
fft.realTransform(output, input as unknown as number[]);
fft.completeSpectrum(output);
const earlyFrameWeight = Math.max(0.25, 1.5 - (hop / totalHops));
for (let bin = 1; bin < FFT_SIZE / 2; bin++) {
const frequency = (bin * sampleRate) / FFT_SIZE;
if (frequency < MIN_ANALYSIS_FREQUENCY || frequency > MAX_ANALYSIS_FREQUENCY) {
continue;
}
const real = output[2 * bin];
const imaginary = output[(2 * bin) + 1];
const magnitude = Math.sqrt((real * real) + (imaginary * imaginary));
if (magnitude < 1e-6) {
continue;
}
const midiPitch = 69 + (12 * Math.log2(frequency / 440));
const roundedPitch = Math.round(midiPitch);
const pitchClass = ((roundedPitch % 12) + 12) % 12;
const centsFromPitchClass = Math.abs(midiPitch - roundedPitch);
const pitchWeight = Math.max(0, 1 - (centsFromPitchClass / 0.5));
const frequencyWeight = 1 / Math.max(frequency, 80);
chroma[pitchClass] += magnitude * magnitude * pitchWeight * frequencyWeight * earlyFrameWeight;
}
}
const chromaTotal = chroma.reduce((sum, value) => sum + value, 0);
if (chromaTotal <= 0) {
return { symbol: 'N', confidence: 0, rms };
}
for (let i = 0; i < chroma.length; i++) {
chroma[i] /= chromaTotal;
}
const { best, second } = buildTriadCandidates(chroma);
return {
symbol: best.symbol,
confidence: clamp(best.score - second.score + (best.score * 0.2), 0, 1),
rms,
};
}
function smoothDetectedChords(results: DetectedAudioChord[]): DetectedAudioChord[] {
if (results.length < 3) {
return results.map(result => ({ ...result }));
}
const smoothed = results.map(result => ({ ...result }));
for (let i = 1; i < smoothed.length - 1; i++) {
const previous = smoothed[i - 1];
const current = smoothed[i];
const next = smoothed[i + 1];
if (current.symbol === 'N') {
continue;
}
if (previous.symbol === next.symbol && previous.symbol !== 'N' && current.symbol !== previous.symbol) {
const surroundingConfidence = Math.max(previous.confidence, next.confidence);
if (current.confidence < surroundingConfidence * 0.85) {
current.symbol = previous.symbol;
current.confidence = Math.max(current.confidence, surroundingConfidence * 0.75);
}
}
}
return smoothed;
}
export function detectChordsFromAudio(
request: AudioChordDetectionRequest,
onProgress?: (progress: AudioChordDetectionProgress) => void,
): DetectedAudioChord[] {
const windows = request.windows.map(cloneWindow);
if (windows.length === 0) {
return [];
}
onProgress?.({
completedWindows: 0,
totalWindows: windows.length,
percent: 0,
});
const rawResults: DetectedAudioChord[] = [];
windows.forEach((window, index) => {
const analysis = analyzeChordWindow(
request.pcm,
request.sampleRate,
request.clipStartOffsetSeconds + (window.startSeconds - request.clipStartOffsetSeconds),
request.clipStartOffsetSeconds + (window.endSeconds - request.clipStartOffsetSeconds),
);
rawResults.push({
barIndex: window.barIndex,
startBeat: window.startBeat,
endBeat: window.endBeat,
symbol: analysis.symbol,
confidence: analysis.confidence,
rms: analysis.rms,
});
onProgress?.({
completedWindows: index + 1,
totalWindows: windows.length,
percent: Math.round(((index + 1) / windows.length) * 100),
});
});
const maxRms = rawResults.reduce((max, result) => Math.max(max, result.rms), 0);
const silenceThreshold = Math.max(ABSOLUTE_SILENCE_RMS, maxRms * RELATIVE_SILENCE_RATIO);
const filtered = rawResults.map(result => (
result.rms < silenceThreshold
? { ...result, symbol: 'N', confidence: 0 }
: result
));
return smoothDetectedChords(filtered);
}
+24
View File
@@ -0,0 +1,24 @@
import {
detectChordsFromAudio,
type AudioChordDetectionProgress,
type AudioChordDetectionRequest,
type DetectedAudioChord,
} from '../util/audioChordDetectionCore';
export type AudioChordDetectionWorkerMessage =
| { type: 'progress'; progress: AudioChordDetectionProgress }
| { type: 'result'; results: DetectedAudioChord[] };
type WorkerScopeLike = typeof globalThis & {
onmessage: ((event: MessageEvent<AudioChordDetectionRequest>) => void) | null;
postMessage: (message: AudioChordDetectionWorkerMessage) => void;
};
const workerScope = self as WorkerScopeLike;
workerScope.onmessage = (event: MessageEvent<AudioChordDetectionRequest>) => {
const results = detectChordsFromAudio(event.data, progress => {
workerScope.postMessage({ type: 'progress', progress });
});
workerScope.postMessage({ type: 'result', results });
};