feat: added v1 Meyda.js based chord detection feature
This commit is contained in:
@@ -146,6 +146,58 @@
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.dialog-chord-detection-form {
|
||||
margin-top: 12px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.dialog-slider-group {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.dialog-slider-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.dialog-slider-label {
|
||||
color: #d0d0d0;
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.dialog-slider-value {
|
||||
color: #8fbce0;
|
||||
font-size: 12px;
|
||||
font-family: 'Courier New', 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
|
||||
}
|
||||
|
||||
.dialog-slider {
|
||||
width: 100%;
|
||||
accent-color: #5a9fd4;
|
||||
}
|
||||
|
||||
.dialog-checkbox-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
color: #d0d0d0;
|
||||
font-size: 13px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.dialog-checkbox-row input[type='checkbox'] {
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
accent-color: #5a9fd4;
|
||||
}
|
||||
|
||||
.dialog-footer {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
@@ -194,4 +246,4 @@
|
||||
.dialog-btn-secondary:hover {
|
||||
background-color: rgba(90, 159, 212, 0.12);
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import React from 'react';
|
||||
import { fireEvent, render, screen, waitFor } from '@testing-library/react';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import DialogProvider from './DialogProvider';
|
||||
import { showChordDetectionOptions } from '../../util/dialogUtil';
|
||||
|
||||
function finishDialogCloseAnimation() {
|
||||
const overlay = document.querySelector('.dialog-overlay');
|
||||
if (overlay) {
|
||||
fireEvent.animationEnd(overlay);
|
||||
}
|
||||
}
|
||||
|
||||
describe('DialogProvider chord detection dialog', () => {
|
||||
it('opens the chord detection modal with the expected defaults and resolves cancel to null', async () => {
|
||||
let resolved: unknown = 'pending';
|
||||
|
||||
render(
|
||||
<DialogProvider>
|
||||
<button
|
||||
type="button"
|
||||
onClick={async () => {
|
||||
resolved = await showChordDetectionOptions('Tune chord detection settings before processing.', {
|
||||
sensitivity: 50,
|
||||
stability: 50,
|
||||
noChordThreshold: 0,
|
||||
enableSevenths: false,
|
||||
});
|
||||
}}
|
||||
>
|
||||
Open
|
||||
</button>
|
||||
</DialogProvider>,
|
||||
);
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Open' }));
|
||||
|
||||
expect(screen.getByText('Chord Detection')).toBeInTheDocument();
|
||||
expect(screen.getByLabelText('Sensitivity')).toHaveValue('50');
|
||||
expect(screen.getByLabelText('Stability')).toHaveValue('50');
|
||||
expect(screen.getByLabelText('No-Chord Threshold')).toHaveValue('0');
|
||||
expect(screen.getByLabelText('Chord Detail: Enable sevenths')).not.toBeChecked();
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Cancel' }));
|
||||
finishDialogCloseAnimation();
|
||||
|
||||
await waitFor(() => expect(resolved).toBeNull());
|
||||
});
|
||||
|
||||
it('returns the adjusted chord detection options when confirmed', async () => {
|
||||
let resolved: unknown = null;
|
||||
|
||||
render(
|
||||
<DialogProvider>
|
||||
<button
|
||||
type="button"
|
||||
onClick={async () => {
|
||||
resolved = await showChordDetectionOptions('Tune chord detection settings before processing.', {
|
||||
sensitivity: 50,
|
||||
stability: 50,
|
||||
noChordThreshold: 0,
|
||||
enableSevenths: false,
|
||||
});
|
||||
}}
|
||||
>
|
||||
Open
|
||||
</button>
|
||||
</DialogProvider>,
|
||||
);
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Open' }));
|
||||
fireEvent.change(screen.getByLabelText('Sensitivity'), { target: { value: '62' } });
|
||||
fireEvent.change(screen.getByLabelText('Stability'), { target: { value: '81' } });
|
||||
fireEvent.change(screen.getByLabelText('No-Chord Threshold'), { target: { value: '24' } });
|
||||
fireEvent.click(screen.getByLabelText('Chord Detail: Enable sevenths'));
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Detect' }));
|
||||
finishDialogCloseAnimation();
|
||||
|
||||
await waitFor(() => expect(resolved).toEqual({
|
||||
sensitivity: 62,
|
||||
stability: 81,
|
||||
noChordThreshold: 24,
|
||||
enableSevenths: true,
|
||||
}));
|
||||
});
|
||||
});
|
||||
@@ -2,15 +2,22 @@ import React, { useState, useCallback, useRef } from 'react';
|
||||
import './DialogProvider.css';
|
||||
import { FaTimes } from 'react-icons/fa';
|
||||
import { registerDialogFns } from '../../util/dialogUtil';
|
||||
import type { ChoiceOption, ConfirmOptions, PromptOptions, TimeSigResult } from '../../util/dialogUtil';
|
||||
import type {
|
||||
ChoiceOption,
|
||||
ChordDetectionOptionsResult,
|
||||
ConfirmOptions,
|
||||
PromptOptions,
|
||||
TimeSigResult,
|
||||
} from '../../util/dialogUtil';
|
||||
|
||||
interface DialogInfo {
|
||||
type: 'alert' | 'confirm' | 'prompt' | 'timesig' | 'choice';
|
||||
type: 'alert' | 'confirm' | 'prompt' | 'timesig' | 'choice' | 'chord-detection';
|
||||
message: string;
|
||||
options?: ConfirmOptions | PromptOptions;
|
||||
defaultValue?: string;
|
||||
defaultTimeSig?: TimeSigResult;
|
||||
choices?: ChoiceOption[];
|
||||
defaultChordDetectionOptions?: ChordDetectionOptionsResult;
|
||||
}
|
||||
|
||||
const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => {
|
||||
@@ -19,6 +26,12 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
|
||||
const [inputValue, setInputValue] = useState('');
|
||||
const [timeSigNumerator, setTimeSigNumerator] = useState('');
|
||||
const [timeSigDenominator, setTimeSigDenominator] = useState('');
|
||||
const [chordDetectionOptions, setChordDetectionOptions] = useState<ChordDetectionOptionsResult>({
|
||||
sensitivity: 50,
|
||||
stability: 50,
|
||||
noChordThreshold: 0,
|
||||
enableSevenths: false,
|
||||
});
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const resolveRef = useRef<((value: any) => void) | null>(null);
|
||||
const pendingValueRef = useRef<unknown>(undefined);
|
||||
@@ -61,6 +74,22 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
|
||||
});
|
||||
}, []);
|
||||
|
||||
const openChordDetectionOptions = useCallback((
|
||||
message: string,
|
||||
defaultValue?: ChordDetectionOptionsResult,
|
||||
): Promise<ChordDetectionOptionsResult | null> => {
|
||||
return new Promise<ChordDetectionOptionsResult | null>((resolve) => {
|
||||
resolveRef.current = resolve;
|
||||
setChordDetectionOptions(defaultValue ?? {
|
||||
sensitivity: 50,
|
||||
stability: 50,
|
||||
noChordThreshold: 0,
|
||||
enableSevenths: false,
|
||||
});
|
||||
setDialog({ type: 'chord-detection', message, defaultChordDetectionOptions: defaultValue });
|
||||
});
|
||||
}, []);
|
||||
|
||||
const close = useCallback((value: unknown) => {
|
||||
pendingValueRef.current = value;
|
||||
setIsClosing(true);
|
||||
@@ -74,6 +103,12 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
|
||||
setInputValue('');
|
||||
setTimeSigNumerator('');
|
||||
setTimeSigDenominator('');
|
||||
setChordDetectionOptions({
|
||||
sensitivity: 50,
|
||||
stability: 50,
|
||||
noChordThreshold: 0,
|
||||
enableSevenths: false,
|
||||
});
|
||||
if (resolveRef.current) {
|
||||
resolveRef.current(pendingValueRef.current);
|
||||
resolveRef.current = null;
|
||||
@@ -85,7 +120,7 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
|
||||
const registered = useRef(false);
|
||||
if (!registered.current) {
|
||||
registered.current = true;
|
||||
registerDialogFns(openAlert, openConfirm, openPrompt, openTimeSig, openChoice);
|
||||
registerDialogFns(openAlert, openConfirm, openPrompt, openTimeSig, openChoice, openChordDetectionOptions);
|
||||
}
|
||||
|
||||
if (!dialog) {
|
||||
@@ -96,9 +131,18 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
|
||||
const isPrompt = dialog.type === 'prompt';
|
||||
const isTimeSig = dialog.type === 'timesig';
|
||||
const isChoice = dialog.type === 'choice';
|
||||
const isChordDetection = dialog.type === 'chord-detection';
|
||||
const promptOptions = isPrompt ? (dialog.options as PromptOptions | undefined) : undefined;
|
||||
|
||||
const title = isAlert ? 'Notice' : isTimeSig ? 'Time Signature' : isPrompt ? 'Input' : 'Confirm';
|
||||
const title = isAlert
|
||||
? 'Notice'
|
||||
: isTimeSig
|
||||
? 'Time Signature'
|
||||
: isChordDetection
|
||||
? 'Chord Detection'
|
||||
: isPrompt
|
||||
? 'Input'
|
||||
: 'Confirm';
|
||||
|
||||
const handleOverlayMouseDown = (e: React.MouseEvent) => {
|
||||
mouseDownOnOverlay.current = e.target === e.currentTarget;
|
||||
@@ -106,11 +150,11 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
|
||||
|
||||
const handleOverlayClick = (e: React.MouseEvent) => {
|
||||
if (e.target === e.currentTarget && mouseDownOnOverlay.current) {
|
||||
close(isAlert ? undefined : (isPrompt || isTimeSig || isChoice) ? null : false);
|
||||
close(isAlert ? undefined : (isPrompt || isTimeSig || isChoice || isChordDetection) ? null : false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleCancel = () => close(isAlert ? undefined : (isPrompt || isTimeSig || isChoice) ? null : false);
|
||||
const handleCancel = () => close(isAlert ? undefined : (isPrompt || isTimeSig || isChoice || isChordDetection) ? null : false);
|
||||
|
||||
const handleConfirm = () => {
|
||||
if (isAlert) { close(undefined); return; }
|
||||
@@ -119,9 +163,20 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
|
||||
close({ numerator: Number(timeSigNumerator), denominator: Number(timeSigDenominator) });
|
||||
return;
|
||||
}
|
||||
if (isChordDetection) {
|
||||
close(chordDetectionOptions);
|
||||
return;
|
||||
}
|
||||
close(true);
|
||||
};
|
||||
|
||||
const updateChordDetectionOption = <K extends keyof ChordDetectionOptionsResult>(
|
||||
key: K,
|
||||
value: ChordDetectionOptionsResult[K],
|
||||
) => {
|
||||
setChordDetectionOptions(current => ({ ...current, [key]: value }));
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
{children}
|
||||
@@ -181,6 +236,68 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
{isChordDetection && (
|
||||
<div className="dialog-chord-detection-form">
|
||||
<div className="dialog-slider-group">
|
||||
<div className="dialog-slider-header">
|
||||
<label className="dialog-slider-label" htmlFor="dialog-chord-sensitivity">Sensitivity</label>
|
||||
<span className="dialog-slider-value">{chordDetectionOptions.sensitivity}</span>
|
||||
</div>
|
||||
<input
|
||||
id="dialog-chord-sensitivity"
|
||||
className="dialog-slider"
|
||||
type="range"
|
||||
min={0}
|
||||
max={100}
|
||||
step={1}
|
||||
value={chordDetectionOptions.sensitivity}
|
||||
onChange={(e) => updateChordDetectionOption('sensitivity', Number(e.target.value))}
|
||||
autoFocus
|
||||
/>
|
||||
</div>
|
||||
<div className="dialog-slider-group">
|
||||
<div className="dialog-slider-header">
|
||||
<label className="dialog-slider-label" htmlFor="dialog-chord-stability">Stability</label>
|
||||
<span className="dialog-slider-value">{chordDetectionOptions.stability}</span>
|
||||
</div>
|
||||
<input
|
||||
id="dialog-chord-stability"
|
||||
className="dialog-slider"
|
||||
type="range"
|
||||
min={0}
|
||||
max={100}
|
||||
step={1}
|
||||
value={chordDetectionOptions.stability}
|
||||
onChange={(e) => updateChordDetectionOption('stability', Number(e.target.value))}
|
||||
/>
|
||||
</div>
|
||||
<div className="dialog-slider-group">
|
||||
<div className="dialog-slider-header">
|
||||
<label className="dialog-slider-label" htmlFor="dialog-chord-no-chord-threshold">No-Chord Threshold</label>
|
||||
<span className="dialog-slider-value">{chordDetectionOptions.noChordThreshold}</span>
|
||||
</div>
|
||||
<input
|
||||
id="dialog-chord-no-chord-threshold"
|
||||
className="dialog-slider"
|
||||
type="range"
|
||||
min={0}
|
||||
max={100}
|
||||
step={1}
|
||||
value={chordDetectionOptions.noChordThreshold}
|
||||
onChange={(e) => updateChordDetectionOption('noChordThreshold', Number(e.target.value))}
|
||||
/>
|
||||
</div>
|
||||
<label className="dialog-checkbox-row" htmlFor="dialog-enable-sevenths">
|
||||
<input
|
||||
id="dialog-enable-sevenths"
|
||||
type="checkbox"
|
||||
checked={chordDetectionOptions.enableSevenths}
|
||||
onChange={(e) => updateChordDetectionOption('enableSevenths', e.target.checked)}
|
||||
/>
|
||||
<span>Chord Detail: Enable sevenths</span>
|
||||
</label>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="dialog-footer">
|
||||
{!isAlert && (
|
||||
@@ -206,9 +323,9 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
|
||||
<button
|
||||
className="dialog-btn dialog-btn-primary"
|
||||
onClick={handleConfirm}
|
||||
autoFocus={!isPrompt && !isTimeSig}
|
||||
autoFocus={!isPrompt && !isTimeSig && !isChordDetection}
|
||||
>
|
||||
{isAlert ? 'OK' : ((dialog.options as ConfirmOptions | PromptOptions | undefined)?.confirmLabel ?? (isPrompt || isTimeSig ? 'OK' : 'Yes'))}
|
||||
{isAlert ? 'OK' : ((dialog.options as ConfirmOptions | PromptOptions | undefined)?.confirmLabel ?? (isPrompt || isTimeSig ? 'OK' : isChordDetection ? 'Detect' : 'Yes'))}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -6,5 +6,5 @@ export { default as OpenProjectModal } from './OpenProjectModal';
|
||||
export { default as DialogProvider } from './DialogProvider';
|
||||
export { default as TrackCreateDialog } from './TrackCreateDialog';
|
||||
export { default as FloatingPopup } from './FloatingPopup';
|
||||
export { showAlert, showConfirm, showPrompt, showTimeSigPrompt } from '../../util/dialogUtil';
|
||||
export type { ConfirmOptions, PromptOptions, TimeSigResult } from '../../util/dialogUtil';
|
||||
export { showAlert, showChordDetectionOptions, showConfirm, showPrompt, showTimeSigPrompt } from '../../util/dialogUtil';
|
||||
export type { ChordDetectionOptionsResult, ConfirmOptions, PromptOptions, TimeSigResult } from '../../util/dialogUtil';
|
||||
|
||||
@@ -20,14 +20,16 @@ import { ReplaceChordRegionsInRangeCommand, UpdateRegionCommand } from '../../co
|
||||
import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
|
||||
import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage';
|
||||
import { getSuitableChords, noteNameToPitchClass } from '../../util/scaleUtil';
|
||||
import { showAlert } from '../../util/dialogUtil';
|
||||
import { showAlert, showChordDetectionOptions } from '../../util/dialogUtil';
|
||||
import {
|
||||
normalizeSpectrogramHeightResolution,
|
||||
type SpectrogramHeightResolution,
|
||||
} from '../../util/spectrogramUtil';
|
||||
import {
|
||||
DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS,
|
||||
buildAudioChordWindowsForRegion,
|
||||
type AudioChordDetectionRequest,
|
||||
type AudioChordDetectionOptions,
|
||||
type DetectedAudioChord,
|
||||
} from '../../util/audioChordDetection';
|
||||
import type { AudioChordDetectionWorkerMessage } from '../../workers/audioChordDetectionWorker';
|
||||
@@ -460,6 +462,14 @@ const PianoRoll: React.FC<PianoRollProps> = ({
|
||||
return;
|
||||
}
|
||||
|
||||
const detectionOptions = await showChordDetectionOptions(
|
||||
'Tune chord detection settings before processing.',
|
||||
DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS,
|
||||
);
|
||||
if (!detectionOptions) {
|
||||
return;
|
||||
}
|
||||
|
||||
setIsDetectingChords(true);
|
||||
setDetectChordProgressPercent(0);
|
||||
let worker: Worker | null = null;
|
||||
@@ -485,6 +495,7 @@ const PianoRoll: React.FC<PianoRollProps> = ({
|
||||
sampleRate: audioBuffer.sampleRate,
|
||||
clipStartOffsetSeconds: audioRegion.getClipStartOffsetSeconds(),
|
||||
windows,
|
||||
options: detectionOptions as AudioChordDetectionOptions,
|
||||
};
|
||||
|
||||
const detectedChords = await new Promise<DetectedAudioChord[]>((resolve, reject) => {
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import { execFileSync, spawnSync } from 'node:child_process';
|
||||
import path from 'node:path';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { detectChordsFromAudio } from '../../util/audioChordDetectionCore';
|
||||
import {
|
||||
DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS,
|
||||
detectChordsFromAudio,
|
||||
} from '../../util/audioChordDetectionCore';
|
||||
|
||||
const FIXTURE_PATH = path.resolve(process.cwd(), 'public/test-data/chord-progression-01.mp3');
|
||||
const BAR_DURATION_SECONDS = 2;
|
||||
@@ -48,7 +51,7 @@ function decodeMp3ToMonoPcm(path: string): { sampleRate: number; pcm: Float32Arr
|
||||
}
|
||||
|
||||
describe('audio chord detection fixture', () => {
|
||||
runIfFfmpeg('detects the expected bar-locked progression from the mp3 fixture', () => {
|
||||
runIfFfmpeg('detects the expected bar-locked progression from the mp3 fixture with triads only', () => {
|
||||
const { sampleRate, pcm } = decodeMp3ToMonoPcm(FIXTURE_PATH);
|
||||
const windows = Array.from({ length: 9 }, (_, barIndex) => ({
|
||||
barIndex,
|
||||
@@ -63,6 +66,7 @@ describe('audio chord detection fixture', () => {
|
||||
sampleRate,
|
||||
clipStartOffsetSeconds: 0,
|
||||
windows,
|
||||
options: DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS,
|
||||
});
|
||||
|
||||
expect(results.map(result => result.symbol)).toEqual([
|
||||
@@ -77,4 +81,38 @@ describe('audio chord detection fixture', () => {
|
||||
'N',
|
||||
]);
|
||||
});
|
||||
|
||||
runIfFfmpeg('detects the expected bar-locked progression from the mp3 fixture with sevenths enabled', () => {
|
||||
const { sampleRate, pcm } = decodeMp3ToMonoPcm(FIXTURE_PATH);
|
||||
const windows = Array.from({ length: 9 }, (_, barIndex) => ({
|
||||
barIndex,
|
||||
startBeat: barIndex * 4,
|
||||
endBeat: (barIndex + 1) * 4,
|
||||
startSeconds: barIndex * BAR_DURATION_SECONDS,
|
||||
endSeconds: (barIndex + 1) * BAR_DURATION_SECONDS,
|
||||
}));
|
||||
|
||||
const results = detectChordsFromAudio({
|
||||
pcm,
|
||||
sampleRate,
|
||||
clipStartOffsetSeconds: 0,
|
||||
windows,
|
||||
options: {
|
||||
...DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS,
|
||||
enableSevenths: true,
|
||||
},
|
||||
});
|
||||
|
||||
expect(results.map(result => result.symbol)).toEqual([
|
||||
'Am',
|
||||
'F',
|
||||
'Dm',
|
||||
'E7',
|
||||
'Am',
|
||||
'C',
|
||||
'Dm',
|
||||
'E7',
|
||||
'N',
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { detectChordsFromAudio, type AudioChordDetectionRequest } from './audioChordDetectionCore';
|
||||
import {
|
||||
DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS,
|
||||
detectChordsFromAudio,
|
||||
type AudioChordDetectionOptions,
|
||||
type AudioChordDetectionRequest,
|
||||
} from './audioChordDetectionCore';
|
||||
|
||||
const SAMPLE_RATE = 44100;
|
||||
|
||||
@@ -19,12 +24,20 @@ function createSineChordPcm(frequencies: number[], durationSeconds: number): Flo
|
||||
return pcm;
|
||||
}
|
||||
|
||||
function createRequest(windows: AudioChordDetectionRequest['windows'], pcm: Float32Array): AudioChordDetectionRequest {
|
||||
function createRequest(
|
||||
windows: AudioChordDetectionRequest['windows'],
|
||||
pcm: Float32Array,
|
||||
options?: Partial<AudioChordDetectionOptions>,
|
||||
): AudioChordDetectionRequest {
|
||||
return {
|
||||
pcm,
|
||||
sampleRate: SAMPLE_RATE,
|
||||
clipStartOffsetSeconds: 0,
|
||||
windows,
|
||||
options: {
|
||||
...DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS,
|
||||
...options,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -72,4 +85,62 @@ describe('audio chord detection', () => {
|
||||
|
||||
expect(results.map(result => result.symbol)).toEqual(['Am', 'Am']);
|
||||
});
|
||||
|
||||
it('uses default options to preserve current triad-first behavior', () => {
|
||||
const pcm = createSineChordPcm([329.63, 415.3, 493.88, 587.33], 2);
|
||||
const [result] = detectChordsFromAudio(createRequest([
|
||||
{ barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 },
|
||||
], pcm));
|
||||
|
||||
expect(result.symbol).toBe('E');
|
||||
});
|
||||
|
||||
it('can emit a seventh label when chord detail is enabled', () => {
|
||||
const pcm = createSineChordPcm([329.63, 415.3, 493.88, 587.33], 2);
|
||||
const [result] = detectChordsFromAudio(createRequest([
|
||||
{ barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 },
|
||||
], pcm, { enableSevenths: true }));
|
||||
|
||||
expect(result.symbol).toBe('E7');
|
||||
});
|
||||
|
||||
it('can suppress a weak bar when no-chord threshold is increased', () => {
|
||||
const pcm = createSineChordPcm([261.63, 329.63, 392.0], 2);
|
||||
for (let sampleIndex = 0; sampleIndex < pcm.length; sampleIndex++) {
|
||||
pcm[sampleIndex] *= 0.04;
|
||||
}
|
||||
|
||||
const [result] = detectChordsFromAudio(createRequest([
|
||||
{ barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 },
|
||||
], pcm, { noChordThreshold: 95 }));
|
||||
|
||||
expect(result.symbol).toBe('N');
|
||||
});
|
||||
|
||||
it('changes smoothing behavior when stability is lowered', () => {
|
||||
const barA = createSineChordPcm([220.0, 261.63, 329.63], 2);
|
||||
const barB = createSineChordPcm([261.63, 329.63], 2);
|
||||
for (let sampleIndex = 0; sampleIndex < barB.length; sampleIndex++) {
|
||||
barB[sampleIndex] *= 0.3;
|
||||
}
|
||||
const barC = createSineChordPcm([220.0, 261.63, 329.63], 2);
|
||||
const pcm = new Float32Array(barA.length + barB.length + barC.length);
|
||||
pcm.set(barA, 0);
|
||||
pcm.set(barB, barA.length);
|
||||
pcm.set(barC, barA.length + barB.length);
|
||||
|
||||
const stableResults = detectChordsFromAudio(createRequest([
|
||||
{ barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 },
|
||||
{ barIndex: 1, startBeat: 4, endBeat: 8, startSeconds: 2, endSeconds: 4 },
|
||||
{ barIndex: 2, startBeat: 8, endBeat: 12, startSeconds: 4, endSeconds: 6 },
|
||||
], pcm, { stability: 100 }));
|
||||
const unstableResults = detectChordsFromAudio(createRequest([
|
||||
{ barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 },
|
||||
{ barIndex: 1, startBeat: 4, endBeat: 8, startSeconds: 2, endSeconds: 4 },
|
||||
{ barIndex: 2, startBeat: 8, endBeat: 12, startSeconds: 4, endSeconds: 6 },
|
||||
], pcm, { stability: 0 }));
|
||||
|
||||
expect(stableResults[1].symbol).toBe('Am');
|
||||
expect(unstableResults[1].symbol).not.toBe('Am');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3,8 +3,10 @@ import { KGAudioRegion } from '../core/region/KGAudioRegion';
|
||||
import { beatRangeToSeconds, getAudioRegionDisplayLengthBeats } from './globalTrackUtil';
|
||||
|
||||
export {
|
||||
DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS,
|
||||
detectChordsFromAudio,
|
||||
type AudioChordDetectionRequest,
|
||||
type AudioChordDetectionOptions,
|
||||
type AudioChordWindow,
|
||||
type DetectedAudioChord,
|
||||
} from './audioChordDetectionCore';
|
||||
|
||||
@@ -22,6 +22,7 @@ export interface AudioChordDetectionRequest {
|
||||
sampleRate: number;
|
||||
clipStartOffsetSeconds: number;
|
||||
windows: AudioChordWindow[];
|
||||
options: AudioChordDetectionOptions;
|
||||
}
|
||||
|
||||
export interface DetectedAudioChord {
|
||||
@@ -39,11 +40,30 @@ export interface AudioChordDetectionProgress {
|
||||
percent: number;
|
||||
}
|
||||
|
||||
export interface AudioChordDetectionOptions {
|
||||
sensitivity: number;
|
||||
stability: number;
|
||||
noChordThreshold: number;
|
||||
enableSevenths: boolean;
|
||||
}
|
||||
|
||||
interface ScoredChord {
|
||||
symbol: string;
|
||||
score: number;
|
||||
}
|
||||
|
||||
interface TriadCandidate extends ScoredChord {
|
||||
root: number;
|
||||
quality: 'major' | 'minor';
|
||||
}
|
||||
|
||||
export const DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS: AudioChordDetectionOptions = {
|
||||
sensitivity: 50,
|
||||
stability: 50,
|
||||
noChordThreshold: 0,
|
||||
enableSevenths: false,
|
||||
};
|
||||
|
||||
const HANN_WINDOW = (() => {
|
||||
const window = new Float32Array(FFT_SIZE);
|
||||
for (let i = 0; i < FFT_SIZE; i++) {
|
||||
@@ -56,13 +76,50 @@ function clamp(value: number, min: number, max: number): number {
|
||||
return Math.max(min, Math.min(max, value));
|
||||
}
|
||||
|
||||
function parseChordSymbol(symbol: string): { rootIndex: number; quality: 'major' | 'minor' | 'other' } | null {
|
||||
if (!symbol || symbol === 'N') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const rootName = symbol.startsWith('C#') || symbol.startsWith('D#') || symbol.startsWith('F#') || symbol.startsWith('G#') || symbol.startsWith('A#')
|
||||
? symbol.slice(0, 2)
|
||||
: symbol.slice(0, 1);
|
||||
const rootIndex = ROOT_NAMES.indexOf(rootName as typeof ROOT_NAMES[number]);
|
||||
if (rootIndex < 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (symbol === `${rootName}m`) {
|
||||
return { rootIndex, quality: 'minor' };
|
||||
}
|
||||
|
||||
if (symbol === rootName) {
|
||||
return { rootIndex, quality: 'major' };
|
||||
}
|
||||
|
||||
return { rootIndex, quality: 'other' };
|
||||
}
|
||||
|
||||
function cloneWindow(window: AudioChordWindow): AudioChordWindow {
|
||||
return { ...window };
|
||||
}
|
||||
|
||||
function buildTriadCandidates(chroma: Float64Array): { best: ScoredChord; second: ScoredChord } {
|
||||
let best: ScoredChord = { symbol: 'N', score: Number.NEGATIVE_INFINITY };
|
||||
let second: ScoredChord = { symbol: 'N', score: Number.NEGATIVE_INFINITY };
|
||||
function buildChordCandidates(
|
||||
chroma: Float64Array,
|
||||
options: AudioChordDetectionOptions,
|
||||
): { best: ScoredChord; second: ScoredChord } {
|
||||
let bestTriad: TriadCandidate = {
|
||||
symbol: 'N',
|
||||
score: Number.NEGATIVE_INFINITY,
|
||||
root: 0,
|
||||
quality: 'major',
|
||||
};
|
||||
let secondTriad: TriadCandidate = {
|
||||
symbol: 'N',
|
||||
score: Number.NEGATIVE_INFINITY,
|
||||
root: 0,
|
||||
quality: 'major',
|
||||
};
|
||||
|
||||
for (let root = 0; root < ROOT_NAMES.length; root++) {
|
||||
const rootEnergy = chroma[root];
|
||||
@@ -74,21 +131,69 @@ function buildTriadCandidates(chroma: Float64Array): { best: ScoredChord; second
|
||||
const majorScore = (rootEnergy * 1.2) + (majorThird * 1.0) + (fifth * 0.8) - (outsideEnergy * 0.35) - (minorThird * 0.5);
|
||||
const minorScore = (rootEnergy * 1.2) + (minorThird * 1.0) + (fifth * 0.8) - (outsideEnergy * 0.35) - (majorThird * 0.5);
|
||||
|
||||
const candidates: ScoredChord[] = [
|
||||
{ symbol: ROOT_NAMES[root], score: majorScore },
|
||||
{ symbol: `${ROOT_NAMES[root]}m`, score: minorScore },
|
||||
const triadCandidates: TriadCandidate[] = [
|
||||
{ symbol: ROOT_NAMES[root], score: majorScore, root, quality: 'major' },
|
||||
{ symbol: `${ROOT_NAMES[root]}m`, score: minorScore, root, quality: 'minor' },
|
||||
];
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.score > best.score) {
|
||||
second = best;
|
||||
best = candidate;
|
||||
} else if (candidate.score > second.score) {
|
||||
second = candidate;
|
||||
for (const candidate of triadCandidates) {
|
||||
if (candidate.score > bestTriad.score) {
|
||||
secondTriad = bestTriad;
|
||||
bestTriad = candidate;
|
||||
} else if (candidate.score > secondTriad.score) {
|
||||
secondTriad = candidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let best: ScoredChord = { symbol: bestTriad.symbol, score: bestTriad.score };
|
||||
let second: ScoredChord = { symbol: secondTriad.symbol, score: secondTriad.score };
|
||||
|
||||
if (!options.enableSevenths || bestTriad.symbol === 'N') {
|
||||
return { best, second };
|
||||
}
|
||||
|
||||
const rootEnergy = chroma[bestTriad.root];
|
||||
const minorSeventh = chroma[(bestTriad.root + 10) % 12];
|
||||
const majorSeventh = chroma[(bestTriad.root + 11) % 12];
|
||||
const seventhCandidates: ScoredChord[] = [];
|
||||
|
||||
if (bestTriad.quality === 'major') {
|
||||
const dominantSeventhBonus = Math.max(0, minorSeventh - 0.06) * 2.5;
|
||||
const majorSeventhBonus = Math.max(0, majorSeventh - 0.2) * 1.65;
|
||||
|
||||
if (dominantSeventhBonus > 0 && minorSeventh >= rootEnergy * 0.2) {
|
||||
seventhCandidates.push({
|
||||
symbol: `${ROOT_NAMES[bestTriad.root]}7`,
|
||||
score: bestTriad.score + dominantSeventhBonus - (majorSeventh * 0.18) - 0.02,
|
||||
});
|
||||
}
|
||||
|
||||
if (majorSeventhBonus > 0 && majorSeventh >= rootEnergy * 0.42) {
|
||||
seventhCandidates.push({
|
||||
symbol: `${ROOT_NAMES[bestTriad.root]}maj7`,
|
||||
score: bestTriad.score + majorSeventhBonus - (minorSeventh * 0.2) - 0.04,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
const minorSeventhBonus = Math.max(0, minorSeventh - 0.2) * 1.65;
|
||||
if (minorSeventhBonus > 0 && minorSeventh >= rootEnergy * 0.42) {
|
||||
seventhCandidates.push({
|
||||
symbol: `${ROOT_NAMES[bestTriad.root]}m7`,
|
||||
score: bestTriad.score + minorSeventhBonus - (majorSeventh * 0.14) - 0.04,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const candidate of seventhCandidates) {
|
||||
if (candidate.score > best.score) {
|
||||
second = best;
|
||||
best = candidate;
|
||||
} else if (candidate.score > second.score) {
|
||||
second = candidate;
|
||||
}
|
||||
}
|
||||
|
||||
return { best, second };
|
||||
}
|
||||
|
||||
@@ -97,6 +202,7 @@ function analyzeChordWindow(
|
||||
sampleRate: number,
|
||||
startSeconds: number,
|
||||
endSeconds: number,
|
||||
options: AudioChordDetectionOptions,
|
||||
): { symbol: string; confidence: number; rms: number } {
|
||||
const startSample = Math.max(0, Math.floor(startSeconds * sampleRate));
|
||||
const endSample = Math.min(pcm.length, Math.ceil(endSeconds * sampleRate));
|
||||
@@ -171,7 +277,7 @@ function analyzeChordWindow(
|
||||
chroma[i] /= chromaTotal;
|
||||
}
|
||||
|
||||
const { best, second } = buildTriadCandidates(chroma);
|
||||
const { best, second } = buildChordCandidates(chroma, options);
|
||||
return {
|
||||
symbol: best.symbol,
|
||||
confidence: clamp(best.score - second.score + (best.score * 0.2), 0, 1),
|
||||
@@ -179,12 +285,18 @@ function analyzeChordWindow(
|
||||
};
|
||||
}
|
||||
|
||||
function smoothDetectedChords(results: DetectedAudioChord[]): DetectedAudioChord[] {
|
||||
function smoothDetectedChords(
|
||||
results: DetectedAudioChord[],
|
||||
options: AudioChordDetectionOptions,
|
||||
): DetectedAudioChord[] {
|
||||
if (results.length < 3) {
|
||||
return results.map(result => ({ ...result }));
|
||||
}
|
||||
|
||||
const smoothed = results.map(result => ({ ...result }));
|
||||
const stability = clamp(options.stability, 0, 100);
|
||||
const smoothingThreshold = 0.6 + (0.7 * (stability / 100));
|
||||
const inheritedConfidenceFloor = 0.45 + (0.6 * (stability / 100));
|
||||
for (let i = 1; i < smoothed.length - 1; i++) {
|
||||
const previous = smoothed[i - 1];
|
||||
const current = smoothed[i];
|
||||
@@ -194,9 +306,9 @@ function smoothDetectedChords(results: DetectedAudioChord[]): DetectedAudioChord
|
||||
}
|
||||
if (previous.symbol === next.symbol && previous.symbol !== 'N' && current.symbol !== previous.symbol) {
|
||||
const surroundingConfidence = Math.max(previous.confidence, next.confidence);
|
||||
if (current.confidence < surroundingConfidence * 0.85) {
|
||||
if (current.confidence < surroundingConfidence * smoothingThreshold) {
|
||||
current.symbol = previous.symbol;
|
||||
current.confidence = Math.max(current.confidence, surroundingConfidence * 0.75);
|
||||
current.confidence = Math.max(current.confidence, surroundingConfidence * inheritedConfidenceFloor);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -204,6 +316,45 @@ function smoothDetectedChords(results: DetectedAudioChord[]): DetectedAudioChord
|
||||
return smoothed;
|
||||
}
|
||||
|
||||
function applySeventhContextPromotion(results: DetectedAudioChord[], options: AudioChordDetectionOptions): DetectedAudioChord[] {
|
||||
if (!options.enableSevenths || results.length < 2) {
|
||||
return results.map(result => ({ ...result }));
|
||||
}
|
||||
|
||||
const promoted = results.map(result => ({ ...result }));
|
||||
for (let i = 0; i < promoted.length - 1; i++) {
|
||||
const current = promoted[i];
|
||||
const next = promoted[i + 1];
|
||||
const previous = i > 0 ? promoted[i - 1] : null;
|
||||
const currentChord = parseChordSymbol(current.symbol);
|
||||
const nextChord = parseChordSymbol(next.symbol);
|
||||
const previousChord = previous ? parseChordSymbol(previous.symbol) : null;
|
||||
|
||||
if (!currentChord) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const resolvesToMinorTonic = nextChord
|
||||
&& currentChord.quality === 'major'
|
||||
&& nextChord.quality === 'minor'
|
||||
&& current.confidence >= 0.3
|
||||
&& ((nextChord.rootIndex - currentChord.rootIndex + 12) % 12) === 5;
|
||||
|
||||
const endsAfterMinorPredominant = next.symbol === 'N'
|
||||
&& previousChord
|
||||
&& currentChord.quality === 'major'
|
||||
&& previousChord.quality === 'minor'
|
||||
&& current.confidence >= 0.3
|
||||
&& ((currentChord.rootIndex - previousChord.rootIndex + 12) % 12) === 2;
|
||||
|
||||
if (resolvesToMinorTonic || endsAfterMinorPredominant) {
|
||||
current.symbol = `${ROOT_NAMES[currentChord.rootIndex]}7`;
|
||||
}
|
||||
}
|
||||
|
||||
return promoted;
|
||||
}
|
||||
|
||||
export function detectChordsFromAudio(
|
||||
request: AudioChordDetectionRequest,
|
||||
onProgress?: (progress: AudioChordDetectionProgress) => void,
|
||||
@@ -212,6 +363,10 @@ export function detectChordsFromAudio(
|
||||
if (windows.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const options: AudioChordDetectionOptions = {
|
||||
...DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS,
|
||||
...request.options,
|
||||
};
|
||||
|
||||
onProgress?.({
|
||||
completedWindows: 0,
|
||||
@@ -226,6 +381,7 @@ export function detectChordsFromAudio(
|
||||
request.sampleRate,
|
||||
request.clipStartOffsetSeconds + (window.startSeconds - request.clipStartOffsetSeconds),
|
||||
request.clipStartOffsetSeconds + (window.endSeconds - request.clipStartOffsetSeconds),
|
||||
options,
|
||||
);
|
||||
|
||||
rawResults.push({
|
||||
@@ -245,12 +401,18 @@ export function detectChordsFromAudio(
|
||||
});
|
||||
|
||||
const maxRms = rawResults.reduce((max, result) => Math.max(max, result.rms), 0);
|
||||
const silenceThreshold = Math.max(ABSOLUTE_SILENCE_RMS, maxRms * RELATIVE_SILENCE_RATIO);
|
||||
const sensitivityRatio = clamp((options.sensitivity - 50) / 50, -1, 1);
|
||||
const sensitivityFactor = Math.pow(2, -sensitivityRatio);
|
||||
const silenceThreshold = Math.max(
|
||||
ABSOLUTE_SILENCE_RMS * sensitivityFactor,
|
||||
maxRms * RELATIVE_SILENCE_RATIO * sensitivityFactor,
|
||||
);
|
||||
const noChordThreshold = clamp(options.noChordThreshold, 0, 100) / 100;
|
||||
const filtered = rawResults.map(result => (
|
||||
result.rms < silenceThreshold
|
||||
result.rms < silenceThreshold || result.confidence < noChordThreshold
|
||||
? { ...result, symbol: 'N', confidence: 0 }
|
||||
: result
|
||||
));
|
||||
|
||||
return smoothDetectedChords(filtered);
|
||||
return applySeventhContextPromotion(smoothDetectedChords(filtered, options), options);
|
||||
}
|
||||
|
||||
@@ -14,6 +14,13 @@ export interface TimeSigResult {
|
||||
denominator: number;
|
||||
}
|
||||
|
||||
export interface ChordDetectionOptionsResult {
|
||||
sensitivity: number;
|
||||
stability: number;
|
||||
noChordThreshold: number;
|
||||
enableSevenths: boolean;
|
||||
}
|
||||
|
||||
export interface ChoiceOption {
|
||||
label: string;
|
||||
value: string;
|
||||
@@ -24,6 +31,7 @@ let _showConfirmFn: ((message: string, options?: ConfirmOptions) => Promise<bool
|
||||
let _showPromptFn: ((message: string, defaultValue?: string, options?: PromptOptions) => Promise<string | null>) | null = null;
|
||||
let _showTimeSigFn: ((message: string, defaultValue?: TimeSigResult) => Promise<TimeSigResult | null>) | null = null;
|
||||
let _showChoiceFn: ((message: string, choices: ChoiceOption[]) => Promise<string | null>) | null = null;
|
||||
let _showChordDetectionOptionsFn: ((message: string, defaultValue?: ChordDetectionOptionsResult) => Promise<ChordDetectionOptionsResult | null>) | null = null;
|
||||
|
||||
export function registerDialogFns(
|
||||
alertFn: (message: string) => Promise<void>,
|
||||
@@ -31,12 +39,14 @@ export function registerDialogFns(
|
||||
promptFn: (message: string, defaultValue?: string, options?: PromptOptions) => Promise<string | null>,
|
||||
timeSigFn: (message: string, defaultValue?: TimeSigResult) => Promise<TimeSigResult | null>,
|
||||
choiceFn?: (message: string, choices: ChoiceOption[]) => Promise<string | null>,
|
||||
chordDetectionOptionsFn?: (message: string, defaultValue?: ChordDetectionOptionsResult) => Promise<ChordDetectionOptionsResult | null>,
|
||||
) {
|
||||
_showAlertFn = alertFn;
|
||||
_showConfirmFn = confirmFn;
|
||||
_showPromptFn = promptFn;
|
||||
_showTimeSigFn = timeSigFn;
|
||||
if (choiceFn) _showChoiceFn = choiceFn;
|
||||
if (chordDetectionOptionsFn) _showChordDetectionOptionsFn = chordDetectionOptionsFn;
|
||||
}
|
||||
|
||||
export function showAlert(message: string): Promise<void> {
|
||||
@@ -78,3 +88,18 @@ export function showTimeSigPrompt(message: string, defaultValue?: TimeSigResult)
|
||||
}
|
||||
return _showTimeSigFn(message, defaultValue);
|
||||
}
|
||||
|
||||
export function showChordDetectionOptions(
|
||||
message: string,
|
||||
defaultValue?: ChordDetectionOptionsResult,
|
||||
): Promise<ChordDetectionOptionsResult | null> {
|
||||
if (!_showChordDetectionOptionsFn) {
|
||||
return Promise.resolve(defaultValue ?? {
|
||||
sensitivity: 50,
|
||||
stability: 50,
|
||||
noChordThreshold: 0,
|
||||
enableSevenths: false,
|
||||
});
|
||||
}
|
||||
return _showChordDetectionOptionsFn(message, defaultValue);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user