feat: added v1 Meyda.js based chord detection feature

This commit is contained in:
Xiaohan-Tian
2026-05-25 20:55:06 -07:00
parent f12add935a
commit a588cb439f
12 changed files with 634 additions and 36 deletions
+34
View File
@@ -227,4 +227,38 @@ Copyright 2019 The MediaPipe Authors
---
---
### 8. Meyda
Meyda is an audio feature extraction library used for chord detection.
Original project: https://github.com/meyda/meyda
```
MIT License
Copyright (c) 2014 Hugh A. Rawlinson, Nevo Segal, Jakub Fiala
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
```
---
Apache License Version 2.0, January 2004: http://www.apache.org/licenses/LICENSE-2.0
+1 -1
View File
@@ -391,4 +391,4 @@ Licensed under the Apache License, Version 2.0, with additional terms (see `LICE
- No patent applications using this software or assets
- Attribution required when used in public/commercial products (“Powered by K.G.Studio”)
Thirdparty notices (e.g., FluidR3_GM SoundFont, midijssoundfonts, VexFlow, prompt structure notes, Gemma 4 E4B, UVR-MDX-NET-Inst_HQ_3, and MediaPipe) are included in `LICENSE`.
Thirdparty notices (e.g., FluidR3_GM SoundFont, midijssoundfonts, VexFlow, prompt structure notes, Gemma 4 E4B, UVR-MDX-NET-Inst_HQ_3, MediaPipe, and Meyda) are included in `LICENSE`.
+53 -1
View File
@@ -146,6 +146,58 @@
line-height: 1;
}
.dialog-chord-detection-form {
margin-top: 12px;
display: flex;
flex-direction: column;
gap: 16px;
}
.dialog-slider-group {
display: flex;
flex-direction: column;
gap: 8px;
}
.dialog-slider-header {
display: flex;
align-items: center;
justify-content: space-between;
gap: 12px;
}
.dialog-slider-label {
color: #d0d0d0;
font-size: 13px;
font-weight: 500;
}
.dialog-slider-value {
color: #8fbce0;
font-size: 12px;
font-family: 'Courier New', 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
}
.dialog-slider {
width: 100%;
accent-color: #5a9fd4;
}
.dialog-checkbox-row {
display: flex;
align-items: center;
gap: 10px;
color: #d0d0d0;
font-size: 13px;
cursor: pointer;
}
.dialog-checkbox-row input[type='checkbox'] {
width: 16px;
height: 16px;
accent-color: #5a9fd4;
}
.dialog-footer {
display: flex;
justify-content: flex-end;
@@ -194,4 +246,4 @@
.dialog-btn-secondary:hover {
background-color: rgba(90, 159, 212, 0.12);
transform: translateY(-1px);
}
}
@@ -0,0 +1,86 @@
import React from 'react';
import { fireEvent, render, screen, waitFor } from '@testing-library/react';
import { describe, expect, it } from 'vitest';
import DialogProvider from './DialogProvider';
import { showChordDetectionOptions } from '../../util/dialogUtil';
function finishDialogCloseAnimation() {
const overlay = document.querySelector('.dialog-overlay');
if (overlay) {
fireEvent.animationEnd(overlay);
}
}
describe('DialogProvider chord detection dialog', () => {
it('opens the chord detection modal with the expected defaults and resolves cancel to null', async () => {
let resolved: unknown = 'pending';
render(
<DialogProvider>
<button
type="button"
onClick={async () => {
resolved = await showChordDetectionOptions('Tune chord detection settings before processing.', {
sensitivity: 50,
stability: 50,
noChordThreshold: 0,
enableSevenths: false,
});
}}
>
Open
</button>
</DialogProvider>,
);
fireEvent.click(screen.getByRole('button', { name: 'Open' }));
expect(screen.getByText('Chord Detection')).toBeInTheDocument();
expect(screen.getByLabelText('Sensitivity')).toHaveValue('50');
expect(screen.getByLabelText('Stability')).toHaveValue('50');
expect(screen.getByLabelText('No-Chord Threshold')).toHaveValue('0');
expect(screen.getByLabelText('Chord Detail: Enable sevenths')).not.toBeChecked();
fireEvent.click(screen.getByRole('button', { name: 'Cancel' }));
finishDialogCloseAnimation();
await waitFor(() => expect(resolved).toBeNull());
});
it('returns the adjusted chord detection options when confirmed', async () => {
let resolved: unknown = null;
render(
<DialogProvider>
<button
type="button"
onClick={async () => {
resolved = await showChordDetectionOptions('Tune chord detection settings before processing.', {
sensitivity: 50,
stability: 50,
noChordThreshold: 0,
enableSevenths: false,
});
}}
>
Open
</button>
</DialogProvider>,
);
fireEvent.click(screen.getByRole('button', { name: 'Open' }));
fireEvent.change(screen.getByLabelText('Sensitivity'), { target: { value: '62' } });
fireEvent.change(screen.getByLabelText('Stability'), { target: { value: '81' } });
fireEvent.change(screen.getByLabelText('No-Chord Threshold'), { target: { value: '24' } });
fireEvent.click(screen.getByLabelText('Chord Detail: Enable sevenths'));
fireEvent.click(screen.getByRole('button', { name: 'Detect' }));
finishDialogCloseAnimation();
await waitFor(() => expect(resolved).toEqual({
sensitivity: 62,
stability: 81,
noChordThreshold: 24,
enableSevenths: true,
}));
});
});
+125 -8
View File
@@ -2,15 +2,22 @@ import React, { useState, useCallback, useRef } from 'react';
import './DialogProvider.css';
import { FaTimes } from 'react-icons/fa';
import { registerDialogFns } from '../../util/dialogUtil';
import type { ChoiceOption, ConfirmOptions, PromptOptions, TimeSigResult } from '../../util/dialogUtil';
import type {
ChoiceOption,
ChordDetectionOptionsResult,
ConfirmOptions,
PromptOptions,
TimeSigResult,
} from '../../util/dialogUtil';
interface DialogInfo {
type: 'alert' | 'confirm' | 'prompt' | 'timesig' | 'choice';
type: 'alert' | 'confirm' | 'prompt' | 'timesig' | 'choice' | 'chord-detection';
message: string;
options?: ConfirmOptions | PromptOptions;
defaultValue?: string;
defaultTimeSig?: TimeSigResult;
choices?: ChoiceOption[];
defaultChordDetectionOptions?: ChordDetectionOptionsResult;
}
const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => {
@@ -19,6 +26,12 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
const [inputValue, setInputValue] = useState('');
const [timeSigNumerator, setTimeSigNumerator] = useState('');
const [timeSigDenominator, setTimeSigDenominator] = useState('');
const [chordDetectionOptions, setChordDetectionOptions] = useState<ChordDetectionOptionsResult>({
sensitivity: 50,
stability: 50,
noChordThreshold: 0,
enableSevenths: false,
});
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const resolveRef = useRef<((value: any) => void) | null>(null);
const pendingValueRef = useRef<unknown>(undefined);
@@ -61,6 +74,22 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
});
}, []);
const openChordDetectionOptions = useCallback((
message: string,
defaultValue?: ChordDetectionOptionsResult,
): Promise<ChordDetectionOptionsResult | null> => {
return new Promise<ChordDetectionOptionsResult | null>((resolve) => {
resolveRef.current = resolve;
setChordDetectionOptions(defaultValue ?? {
sensitivity: 50,
stability: 50,
noChordThreshold: 0,
enableSevenths: false,
});
setDialog({ type: 'chord-detection', message, defaultChordDetectionOptions: defaultValue });
});
}, []);
const close = useCallback((value: unknown) => {
pendingValueRef.current = value;
setIsClosing(true);
@@ -74,6 +103,12 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
setInputValue('');
setTimeSigNumerator('');
setTimeSigDenominator('');
setChordDetectionOptions({
sensitivity: 50,
stability: 50,
noChordThreshold: 0,
enableSevenths: false,
});
if (resolveRef.current) {
resolveRef.current(pendingValueRef.current);
resolveRef.current = null;
@@ -85,7 +120,7 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
const registered = useRef(false);
if (!registered.current) {
registered.current = true;
registerDialogFns(openAlert, openConfirm, openPrompt, openTimeSig, openChoice);
registerDialogFns(openAlert, openConfirm, openPrompt, openTimeSig, openChoice, openChordDetectionOptions);
}
if (!dialog) {
@@ -96,9 +131,18 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
const isPrompt = dialog.type === 'prompt';
const isTimeSig = dialog.type === 'timesig';
const isChoice = dialog.type === 'choice';
const isChordDetection = dialog.type === 'chord-detection';
const promptOptions = isPrompt ? (dialog.options as PromptOptions | undefined) : undefined;
const title = isAlert ? 'Notice' : isTimeSig ? 'Time Signature' : isPrompt ? 'Input' : 'Confirm';
const title = isAlert
? 'Notice'
: isTimeSig
? 'Time Signature'
: isChordDetection
? 'Chord Detection'
: isPrompt
? 'Input'
: 'Confirm';
const handleOverlayMouseDown = (e: React.MouseEvent) => {
mouseDownOnOverlay.current = e.target === e.currentTarget;
@@ -106,11 +150,11 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
const handleOverlayClick = (e: React.MouseEvent) => {
if (e.target === e.currentTarget && mouseDownOnOverlay.current) {
close(isAlert ? undefined : (isPrompt || isTimeSig || isChoice) ? null : false);
close(isAlert ? undefined : (isPrompt || isTimeSig || isChoice || isChordDetection) ? null : false);
}
};
const handleCancel = () => close(isAlert ? undefined : (isPrompt || isTimeSig || isChoice) ? null : false);
const handleCancel = () => close(isAlert ? undefined : (isPrompt || isTimeSig || isChoice || isChordDetection) ? null : false);
const handleConfirm = () => {
if (isAlert) { close(undefined); return; }
@@ -119,9 +163,20 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
close({ numerator: Number(timeSigNumerator), denominator: Number(timeSigDenominator) });
return;
}
if (isChordDetection) {
close(chordDetectionOptions);
return;
}
close(true);
};
const updateChordDetectionOption = <K extends keyof ChordDetectionOptionsResult>(
key: K,
value: ChordDetectionOptionsResult[K],
) => {
setChordDetectionOptions(current => ({ ...current, [key]: value }));
};
return (
<>
{children}
@@ -181,6 +236,68 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
/>
</div>
)}
{isChordDetection && (
<div className="dialog-chord-detection-form">
<div className="dialog-slider-group">
<div className="dialog-slider-header">
<label className="dialog-slider-label" htmlFor="dialog-chord-sensitivity">Sensitivity</label>
<span className="dialog-slider-value">{chordDetectionOptions.sensitivity}</span>
</div>
<input
id="dialog-chord-sensitivity"
className="dialog-slider"
type="range"
min={0}
max={100}
step={1}
value={chordDetectionOptions.sensitivity}
onChange={(e) => updateChordDetectionOption('sensitivity', Number(e.target.value))}
autoFocus
/>
</div>
<div className="dialog-slider-group">
<div className="dialog-slider-header">
<label className="dialog-slider-label" htmlFor="dialog-chord-stability">Stability</label>
<span className="dialog-slider-value">{chordDetectionOptions.stability}</span>
</div>
<input
id="dialog-chord-stability"
className="dialog-slider"
type="range"
min={0}
max={100}
step={1}
value={chordDetectionOptions.stability}
onChange={(e) => updateChordDetectionOption('stability', Number(e.target.value))}
/>
</div>
<div className="dialog-slider-group">
<div className="dialog-slider-header">
<label className="dialog-slider-label" htmlFor="dialog-chord-no-chord-threshold">No-Chord Threshold</label>
<span className="dialog-slider-value">{chordDetectionOptions.noChordThreshold}</span>
</div>
<input
id="dialog-chord-no-chord-threshold"
className="dialog-slider"
type="range"
min={0}
max={100}
step={1}
value={chordDetectionOptions.noChordThreshold}
onChange={(e) => updateChordDetectionOption('noChordThreshold', Number(e.target.value))}
/>
</div>
<label className="dialog-checkbox-row" htmlFor="dialog-enable-sevenths">
<input
id="dialog-enable-sevenths"
type="checkbox"
checked={chordDetectionOptions.enableSevenths}
onChange={(e) => updateChordDetectionOption('enableSevenths', e.target.checked)}
/>
<span>Chord Detail: Enable sevenths</span>
</label>
</div>
)}
</div>
<div className="dialog-footer">
{!isAlert && (
@@ -206,9 +323,9 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
<button
className="dialog-btn dialog-btn-primary"
onClick={handleConfirm}
autoFocus={!isPrompt && !isTimeSig}
autoFocus={!isPrompt && !isTimeSig && !isChordDetection}
>
{isAlert ? 'OK' : ((dialog.options as ConfirmOptions | PromptOptions | undefined)?.confirmLabel ?? (isPrompt || isTimeSig ? 'OK' : 'Yes'))}
{isAlert ? 'OK' : ((dialog.options as ConfirmOptions | PromptOptions | undefined)?.confirmLabel ?? (isPrompt || isTimeSig ? 'OK' : isChordDetection ? 'Detect' : 'Yes'))}
</button>
)}
</div>
+2 -2
View File
@@ -6,5 +6,5 @@ export { default as OpenProjectModal } from './OpenProjectModal';
export { default as DialogProvider } from './DialogProvider';
export { default as TrackCreateDialog } from './TrackCreateDialog';
export { default as FloatingPopup } from './FloatingPopup';
export { showAlert, showConfirm, showPrompt, showTimeSigPrompt } from '../../util/dialogUtil';
export type { ConfirmOptions, PromptOptions, TimeSigResult } from '../../util/dialogUtil';
export { showAlert, showChordDetectionOptions, showConfirm, showPrompt, showTimeSigPrompt } from '../../util/dialogUtil';
export type { ChordDetectionOptionsResult, ConfirmOptions, PromptOptions, TimeSigResult } from '../../util/dialogUtil';
+12 -1
View File
@@ -20,14 +20,16 @@ import { ReplaceChordRegionsInRangeCommand, UpdateRegionCommand } from '../../co
import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage';
import { getSuitableChords, noteNameToPitchClass } from '../../util/scaleUtil';
import { showAlert } from '../../util/dialogUtil';
import { showAlert, showChordDetectionOptions } from '../../util/dialogUtil';
import {
normalizeSpectrogramHeightResolution,
type SpectrogramHeightResolution,
} from '../../util/spectrogramUtil';
import {
DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS,
buildAudioChordWindowsForRegion,
type AudioChordDetectionRequest,
type AudioChordDetectionOptions,
type DetectedAudioChord,
} from '../../util/audioChordDetection';
import type { AudioChordDetectionWorkerMessage } from '../../workers/audioChordDetectionWorker';
@@ -460,6 +462,14 @@ const PianoRoll: React.FC<PianoRollProps> = ({
return;
}
const detectionOptions = await showChordDetectionOptions(
'Tune chord detection settings before processing.',
DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS,
);
if (!detectionOptions) {
return;
}
setIsDetectingChords(true);
setDetectChordProgressPercent(0);
let worker: Worker | null = null;
@@ -485,6 +495,7 @@ const PianoRoll: React.FC<PianoRollProps> = ({
sampleRate: audioBuffer.sampleRate,
clipStartOffsetSeconds: audioRegion.getClipStartOffsetSeconds(),
windows,
options: detectionOptions as AudioChordDetectionOptions,
};
const detectedChords = await new Promise<DetectedAudioChord[]>((resolve, reject) => {
@@ -1,7 +1,10 @@
import { execFileSync, spawnSync } from 'node:child_process';
import path from 'node:path';
import { describe, expect, it } from 'vitest';
import { detectChordsFromAudio } from '../../util/audioChordDetectionCore';
import {
DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS,
detectChordsFromAudio,
} from '../../util/audioChordDetectionCore';
const FIXTURE_PATH = path.resolve(process.cwd(), 'public/test-data/chord-progression-01.mp3');
const BAR_DURATION_SECONDS = 2;
@@ -48,7 +51,7 @@ function decodeMp3ToMonoPcm(path: string): { sampleRate: number; pcm: Float32Arr
}
describe('audio chord detection fixture', () => {
runIfFfmpeg('detects the expected bar-locked progression from the mp3 fixture', () => {
runIfFfmpeg('detects the expected bar-locked progression from the mp3 fixture with triads only', () => {
const { sampleRate, pcm } = decodeMp3ToMonoPcm(FIXTURE_PATH);
const windows = Array.from({ length: 9 }, (_, barIndex) => ({
barIndex,
@@ -63,6 +66,7 @@ describe('audio chord detection fixture', () => {
sampleRate,
clipStartOffsetSeconds: 0,
windows,
options: DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS,
});
expect(results.map(result => result.symbol)).toEqual([
@@ -77,4 +81,38 @@ describe('audio chord detection fixture', () => {
'N',
]);
});
runIfFfmpeg('detects the expected bar-locked progression from the mp3 fixture with sevenths enabled', () => {
const { sampleRate, pcm } = decodeMp3ToMonoPcm(FIXTURE_PATH);
const windows = Array.from({ length: 9 }, (_, barIndex) => ({
barIndex,
startBeat: barIndex * 4,
endBeat: (barIndex + 1) * 4,
startSeconds: barIndex * BAR_DURATION_SECONDS,
endSeconds: (barIndex + 1) * BAR_DURATION_SECONDS,
}));
const results = detectChordsFromAudio({
pcm,
sampleRate,
clipStartOffsetSeconds: 0,
windows,
options: {
...DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS,
enableSevenths: true,
},
});
expect(results.map(result => result.symbol)).toEqual([
'Am',
'F',
'Dm',
'E7',
'Am',
'C',
'Dm',
'E7',
'N',
]);
});
});
+73 -2
View File
@@ -1,5 +1,10 @@
import { describe, expect, it } from 'vitest';
import { detectChordsFromAudio, type AudioChordDetectionRequest } from './audioChordDetectionCore';
import {
DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS,
detectChordsFromAudio,
type AudioChordDetectionOptions,
type AudioChordDetectionRequest,
} from './audioChordDetectionCore';
const SAMPLE_RATE = 44100;
@@ -19,12 +24,20 @@ function createSineChordPcm(frequencies: number[], durationSeconds: number): Flo
return pcm;
}
function createRequest(windows: AudioChordDetectionRequest['windows'], pcm: Float32Array): AudioChordDetectionRequest {
function createRequest(
windows: AudioChordDetectionRequest['windows'],
pcm: Float32Array,
options?: Partial<AudioChordDetectionOptions>,
): AudioChordDetectionRequest {
return {
pcm,
sampleRate: SAMPLE_RATE,
clipStartOffsetSeconds: 0,
windows,
options: {
...DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS,
...options,
},
};
}
@@ -72,4 +85,62 @@ describe('audio chord detection', () => {
expect(results.map(result => result.symbol)).toEqual(['Am', 'Am']);
});
it('uses default options to preserve current triad-first behavior', () => {
const pcm = createSineChordPcm([329.63, 415.3, 493.88, 587.33], 2);
const [result] = detectChordsFromAudio(createRequest([
{ barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 },
], pcm));
expect(result.symbol).toBe('E');
});
it('can emit a seventh label when chord detail is enabled', () => {
const pcm = createSineChordPcm([329.63, 415.3, 493.88, 587.33], 2);
const [result] = detectChordsFromAudio(createRequest([
{ barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 },
], pcm, { enableSevenths: true }));
expect(result.symbol).toBe('E7');
});
it('can suppress a weak bar when no-chord threshold is increased', () => {
const pcm = createSineChordPcm([261.63, 329.63, 392.0], 2);
for (let sampleIndex = 0; sampleIndex < pcm.length; sampleIndex++) {
pcm[sampleIndex] *= 0.04;
}
const [result] = detectChordsFromAudio(createRequest([
{ barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 },
], pcm, { noChordThreshold: 95 }));
expect(result.symbol).toBe('N');
});
it('changes smoothing behavior when stability is lowered', () => {
const barA = createSineChordPcm([220.0, 261.63, 329.63], 2);
const barB = createSineChordPcm([261.63, 329.63], 2);
for (let sampleIndex = 0; sampleIndex < barB.length; sampleIndex++) {
barB[sampleIndex] *= 0.3;
}
const barC = createSineChordPcm([220.0, 261.63, 329.63], 2);
const pcm = new Float32Array(barA.length + barB.length + barC.length);
pcm.set(barA, 0);
pcm.set(barB, barA.length);
pcm.set(barC, barA.length + barB.length);
const stableResults = detectChordsFromAudio(createRequest([
{ barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 },
{ barIndex: 1, startBeat: 4, endBeat: 8, startSeconds: 2, endSeconds: 4 },
{ barIndex: 2, startBeat: 8, endBeat: 12, startSeconds: 4, endSeconds: 6 },
], pcm, { stability: 100 }));
const unstableResults = detectChordsFromAudio(createRequest([
{ barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 },
{ barIndex: 1, startBeat: 4, endBeat: 8, startSeconds: 2, endSeconds: 4 },
{ barIndex: 2, startBeat: 8, endBeat: 12, startSeconds: 4, endSeconds: 6 },
], pcm, { stability: 0 }));
expect(stableResults[1].symbol).toBe('Am');
expect(unstableResults[1].symbol).not.toBe('Am');
});
});
+2
View File
@@ -3,8 +3,10 @@ import { KGAudioRegion } from '../core/region/KGAudioRegion';
import { beatRangeToSeconds, getAudioRegionDisplayLengthBeats } from './globalTrackUtil';
export {
DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS,
detectChordsFromAudio,
type AudioChordDetectionRequest,
type AudioChordDetectionOptions,
type AudioChordWindow,
type DetectedAudioChord,
} from './audioChordDetectionCore';
+181 -19
View File
@@ -22,6 +22,7 @@ export interface AudioChordDetectionRequest {
sampleRate: number;
clipStartOffsetSeconds: number;
windows: AudioChordWindow[];
options: AudioChordDetectionOptions;
}
export interface DetectedAudioChord {
@@ -39,11 +40,30 @@ export interface AudioChordDetectionProgress {
percent: number;
}
export interface AudioChordDetectionOptions {
sensitivity: number;
stability: number;
noChordThreshold: number;
enableSevenths: boolean;
}
interface ScoredChord {
symbol: string;
score: number;
}
interface TriadCandidate extends ScoredChord {
root: number;
quality: 'major' | 'minor';
}
export const DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS: AudioChordDetectionOptions = {
sensitivity: 50,
stability: 50,
noChordThreshold: 0,
enableSevenths: false,
};
const HANN_WINDOW = (() => {
const window = new Float32Array(FFT_SIZE);
for (let i = 0; i < FFT_SIZE; i++) {
@@ -56,13 +76,50 @@ function clamp(value: number, min: number, max: number): number {
return Math.max(min, Math.min(max, value));
}
function parseChordSymbol(symbol: string): { rootIndex: number; quality: 'major' | 'minor' | 'other' } | null {
if (!symbol || symbol === 'N') {
return null;
}
const rootName = symbol.startsWith('C#') || symbol.startsWith('D#') || symbol.startsWith('F#') || symbol.startsWith('G#') || symbol.startsWith('A#')
? symbol.slice(0, 2)
: symbol.slice(0, 1);
const rootIndex = ROOT_NAMES.indexOf(rootName as typeof ROOT_NAMES[number]);
if (rootIndex < 0) {
return null;
}
if (symbol === `${rootName}m`) {
return { rootIndex, quality: 'minor' };
}
if (symbol === rootName) {
return { rootIndex, quality: 'major' };
}
return { rootIndex, quality: 'other' };
}
function cloneWindow(window: AudioChordWindow): AudioChordWindow {
return { ...window };
}
function buildTriadCandidates(chroma: Float64Array): { best: ScoredChord; second: ScoredChord } {
let best: ScoredChord = { symbol: 'N', score: Number.NEGATIVE_INFINITY };
let second: ScoredChord = { symbol: 'N', score: Number.NEGATIVE_INFINITY };
function buildChordCandidates(
chroma: Float64Array,
options: AudioChordDetectionOptions,
): { best: ScoredChord; second: ScoredChord } {
let bestTriad: TriadCandidate = {
symbol: 'N',
score: Number.NEGATIVE_INFINITY,
root: 0,
quality: 'major',
};
let secondTriad: TriadCandidate = {
symbol: 'N',
score: Number.NEGATIVE_INFINITY,
root: 0,
quality: 'major',
};
for (let root = 0; root < ROOT_NAMES.length; root++) {
const rootEnergy = chroma[root];
@@ -74,21 +131,69 @@ function buildTriadCandidates(chroma: Float64Array): { best: ScoredChord; second
const majorScore = (rootEnergy * 1.2) + (majorThird * 1.0) + (fifth * 0.8) - (outsideEnergy * 0.35) - (minorThird * 0.5);
const minorScore = (rootEnergy * 1.2) + (minorThird * 1.0) + (fifth * 0.8) - (outsideEnergy * 0.35) - (majorThird * 0.5);
const candidates: ScoredChord[] = [
{ symbol: ROOT_NAMES[root], score: majorScore },
{ symbol: `${ROOT_NAMES[root]}m`, score: minorScore },
const triadCandidates: TriadCandidate[] = [
{ symbol: ROOT_NAMES[root], score: majorScore, root, quality: 'major' },
{ symbol: `${ROOT_NAMES[root]}m`, score: minorScore, root, quality: 'minor' },
];
for (const candidate of candidates) {
if (candidate.score > best.score) {
second = best;
best = candidate;
} else if (candidate.score > second.score) {
second = candidate;
for (const candidate of triadCandidates) {
if (candidate.score > bestTriad.score) {
secondTriad = bestTriad;
bestTriad = candidate;
} else if (candidate.score > secondTriad.score) {
secondTriad = candidate;
}
}
}
let best: ScoredChord = { symbol: bestTriad.symbol, score: bestTriad.score };
let second: ScoredChord = { symbol: secondTriad.symbol, score: secondTriad.score };
if (!options.enableSevenths || bestTriad.symbol === 'N') {
return { best, second };
}
const rootEnergy = chroma[bestTriad.root];
const minorSeventh = chroma[(bestTriad.root + 10) % 12];
const majorSeventh = chroma[(bestTriad.root + 11) % 12];
const seventhCandidates: ScoredChord[] = [];
if (bestTriad.quality === 'major') {
const dominantSeventhBonus = Math.max(0, minorSeventh - 0.06) * 2.5;
const majorSeventhBonus = Math.max(0, majorSeventh - 0.2) * 1.65;
if (dominantSeventhBonus > 0 && minorSeventh >= rootEnergy * 0.2) {
seventhCandidates.push({
symbol: `${ROOT_NAMES[bestTriad.root]}7`,
score: bestTriad.score + dominantSeventhBonus - (majorSeventh * 0.18) - 0.02,
});
}
if (majorSeventhBonus > 0 && majorSeventh >= rootEnergy * 0.42) {
seventhCandidates.push({
symbol: `${ROOT_NAMES[bestTriad.root]}maj7`,
score: bestTriad.score + majorSeventhBonus - (minorSeventh * 0.2) - 0.04,
});
}
} else {
const minorSeventhBonus = Math.max(0, minorSeventh - 0.2) * 1.65;
if (minorSeventhBonus > 0 && minorSeventh >= rootEnergy * 0.42) {
seventhCandidates.push({
symbol: `${ROOT_NAMES[bestTriad.root]}m7`,
score: bestTriad.score + minorSeventhBonus - (majorSeventh * 0.14) - 0.04,
});
}
}
for (const candidate of seventhCandidates) {
if (candidate.score > best.score) {
second = best;
best = candidate;
} else if (candidate.score > second.score) {
second = candidate;
}
}
return { best, second };
}
@@ -97,6 +202,7 @@ function analyzeChordWindow(
sampleRate: number,
startSeconds: number,
endSeconds: number,
options: AudioChordDetectionOptions,
): { symbol: string; confidence: number; rms: number } {
const startSample = Math.max(0, Math.floor(startSeconds * sampleRate));
const endSample = Math.min(pcm.length, Math.ceil(endSeconds * sampleRate));
@@ -171,7 +277,7 @@ function analyzeChordWindow(
chroma[i] /= chromaTotal;
}
const { best, second } = buildTriadCandidates(chroma);
const { best, second } = buildChordCandidates(chroma, options);
return {
symbol: best.symbol,
confidence: clamp(best.score - second.score + (best.score * 0.2), 0, 1),
@@ -179,12 +285,18 @@ function analyzeChordWindow(
};
}
function smoothDetectedChords(results: DetectedAudioChord[]): DetectedAudioChord[] {
function smoothDetectedChords(
results: DetectedAudioChord[],
options: AudioChordDetectionOptions,
): DetectedAudioChord[] {
if (results.length < 3) {
return results.map(result => ({ ...result }));
}
const smoothed = results.map(result => ({ ...result }));
const stability = clamp(options.stability, 0, 100);
const smoothingThreshold = 0.6 + (0.7 * (stability / 100));
const inheritedConfidenceFloor = 0.45 + (0.6 * (stability / 100));
for (let i = 1; i < smoothed.length - 1; i++) {
const previous = smoothed[i - 1];
const current = smoothed[i];
@@ -194,9 +306,9 @@ function smoothDetectedChords(results: DetectedAudioChord[]): DetectedAudioChord
}
if (previous.symbol === next.symbol && previous.symbol !== 'N' && current.symbol !== previous.symbol) {
const surroundingConfidence = Math.max(previous.confidence, next.confidence);
if (current.confidence < surroundingConfidence * 0.85) {
if (current.confidence < surroundingConfidence * smoothingThreshold) {
current.symbol = previous.symbol;
current.confidence = Math.max(current.confidence, surroundingConfidence * 0.75);
current.confidence = Math.max(current.confidence, surroundingConfidence * inheritedConfidenceFloor);
}
}
}
@@ -204,6 +316,45 @@ function smoothDetectedChords(results: DetectedAudioChord[]): DetectedAudioChord
return smoothed;
}
function applySeventhContextPromotion(results: DetectedAudioChord[], options: AudioChordDetectionOptions): DetectedAudioChord[] {
if (!options.enableSevenths || results.length < 2) {
return results.map(result => ({ ...result }));
}
const promoted = results.map(result => ({ ...result }));
for (let i = 0; i < promoted.length - 1; i++) {
const current = promoted[i];
const next = promoted[i + 1];
const previous = i > 0 ? promoted[i - 1] : null;
const currentChord = parseChordSymbol(current.symbol);
const nextChord = parseChordSymbol(next.symbol);
const previousChord = previous ? parseChordSymbol(previous.symbol) : null;
if (!currentChord) {
continue;
}
const resolvesToMinorTonic = nextChord
&& currentChord.quality === 'major'
&& nextChord.quality === 'minor'
&& current.confidence >= 0.3
&& ((nextChord.rootIndex - currentChord.rootIndex + 12) % 12) === 5;
const endsAfterMinorPredominant = next.symbol === 'N'
&& previousChord
&& currentChord.quality === 'major'
&& previousChord.quality === 'minor'
&& current.confidence >= 0.3
&& ((currentChord.rootIndex - previousChord.rootIndex + 12) % 12) === 2;
if (resolvesToMinorTonic || endsAfterMinorPredominant) {
current.symbol = `${ROOT_NAMES[currentChord.rootIndex]}7`;
}
}
return promoted;
}
export function detectChordsFromAudio(
request: AudioChordDetectionRequest,
onProgress?: (progress: AudioChordDetectionProgress) => void,
@@ -212,6 +363,10 @@ export function detectChordsFromAudio(
if (windows.length === 0) {
return [];
}
const options: AudioChordDetectionOptions = {
...DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS,
...request.options,
};
onProgress?.({
completedWindows: 0,
@@ -226,6 +381,7 @@ export function detectChordsFromAudio(
request.sampleRate,
request.clipStartOffsetSeconds + (window.startSeconds - request.clipStartOffsetSeconds),
request.clipStartOffsetSeconds + (window.endSeconds - request.clipStartOffsetSeconds),
options,
);
rawResults.push({
@@ -245,12 +401,18 @@ export function detectChordsFromAudio(
});
const maxRms = rawResults.reduce((max, result) => Math.max(max, result.rms), 0);
const silenceThreshold = Math.max(ABSOLUTE_SILENCE_RMS, maxRms * RELATIVE_SILENCE_RATIO);
const sensitivityRatio = clamp((options.sensitivity - 50) / 50, -1, 1);
const sensitivityFactor = Math.pow(2, -sensitivityRatio);
const silenceThreshold = Math.max(
ABSOLUTE_SILENCE_RMS * sensitivityFactor,
maxRms * RELATIVE_SILENCE_RATIO * sensitivityFactor,
);
const noChordThreshold = clamp(options.noChordThreshold, 0, 100) / 100;
const filtered = rawResults.map(result => (
result.rms < silenceThreshold
result.rms < silenceThreshold || result.confidence < noChordThreshold
? { ...result, symbol: 'N', confidence: 0 }
: result
));
return smoothDetectedChords(filtered);
return applySeventhContextPromotion(smoothDetectedChords(filtered, options), options);
}
+25
View File
@@ -14,6 +14,13 @@ export interface TimeSigResult {
denominator: number;
}
export interface ChordDetectionOptionsResult {
sensitivity: number;
stability: number;
noChordThreshold: number;
enableSevenths: boolean;
}
export interface ChoiceOption {
label: string;
value: string;
@@ -24,6 +31,7 @@ let _showConfirmFn: ((message: string, options?: ConfirmOptions) => Promise<bool
let _showPromptFn: ((message: string, defaultValue?: string, options?: PromptOptions) => Promise<string | null>) | null = null;
let _showTimeSigFn: ((message: string, defaultValue?: TimeSigResult) => Promise<TimeSigResult | null>) | null = null;
let _showChoiceFn: ((message: string, choices: ChoiceOption[]) => Promise<string | null>) | null = null;
let _showChordDetectionOptionsFn: ((message: string, defaultValue?: ChordDetectionOptionsResult) => Promise<ChordDetectionOptionsResult | null>) | null = null;
export function registerDialogFns(
alertFn: (message: string) => Promise<void>,
@@ -31,12 +39,14 @@ export function registerDialogFns(
promptFn: (message: string, defaultValue?: string, options?: PromptOptions) => Promise<string | null>,
timeSigFn: (message: string, defaultValue?: TimeSigResult) => Promise<TimeSigResult | null>,
choiceFn?: (message: string, choices: ChoiceOption[]) => Promise<string | null>,
chordDetectionOptionsFn?: (message: string, defaultValue?: ChordDetectionOptionsResult) => Promise<ChordDetectionOptionsResult | null>,
) {
_showAlertFn = alertFn;
_showConfirmFn = confirmFn;
_showPromptFn = promptFn;
_showTimeSigFn = timeSigFn;
if (choiceFn) _showChoiceFn = choiceFn;
if (chordDetectionOptionsFn) _showChordDetectionOptionsFn = chordDetectionOptionsFn;
}
export function showAlert(message: string): Promise<void> {
@@ -78,3 +88,18 @@ export function showTimeSigPrompt(message: string, defaultValue?: TimeSigResult)
}
return _showTimeSigFn(message, defaultValue);
}
export function showChordDetectionOptions(
message: string,
defaultValue?: ChordDetectionOptionsResult,
): Promise<ChordDetectionOptionsResult | null> {
if (!_showChordDetectionOptionsFn) {
return Promise.resolve(defaultValue ?? {
sensitivity: 50,
stability: 50,
noChordThreshold: 0,
enableSevenths: false,
});
}
return _showChordDetectionOptionsFn(message, defaultValue);
}