diff --git a/LICENSE b/LICENSE index 6bf1f7b..8a4ce5a 100644 --- a/LICENSE +++ b/LICENSE @@ -227,4 +227,38 @@ Copyright 2019 The MediaPipe Authors --- +--- + +### 8. Meyda + +Meyda is an audio feature extraction library used for chord detection. + +Original project: https://github.com/meyda/meyda + +``` +MIT License + +Copyright (c) 2014 Hugh A. Rawlinson, Nevo Segal, Jakub Fiala + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +``` + +--- + Apache License Version 2.0, January 2004: http://www.apache.org/licenses/LICENSE-2.0 \ No newline at end of file diff --git a/README.md b/README.md index 892d223..568456f 100644 --- a/README.md +++ b/README.md @@ -391,4 +391,4 @@ Licensed under the Apache License, Version 2.0, with additional terms (see `LICE - No patent applications using this software or assets - Attribution required when used in public/commercial products (“Powered by K.G.Studio”) -Third‑party notices (e.g., FluidR3_GM SoundFont, midi‑js‑soundfonts, VexFlow, prompt structure notes, Gemma 4 E4B, UVR-MDX-NET-Inst_HQ_3, and MediaPipe) are included in `LICENSE`. +Third‑party notices (e.g., FluidR3_GM SoundFont, midi‑js‑soundfonts, VexFlow, prompt structure notes, Gemma 4 E4B, UVR-MDX-NET-Inst_HQ_3, MediaPipe, and Meyda) are included in `LICENSE`. diff --git a/src/components/common/DialogProvider.css b/src/components/common/DialogProvider.css index 2ecc648..1321ac1 100644 --- a/src/components/common/DialogProvider.css +++ b/src/components/common/DialogProvider.css @@ -146,6 +146,58 @@ line-height: 1; } +.dialog-chord-detection-form { + margin-top: 12px; + display: flex; + flex-direction: column; + gap: 16px; +} + +.dialog-slider-group { + display: flex; + flex-direction: column; + gap: 8px; +} + +.dialog-slider-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; +} + +.dialog-slider-label { + color: #d0d0d0; + font-size: 13px; + font-weight: 500; +} + +.dialog-slider-value { + color: #8fbce0; + font-size: 12px; + font-family: 'Courier New', 'Monaco', 'Menlo', 'Ubuntu Mono', monospace; +} + +.dialog-slider { + width: 100%; + accent-color: #5a9fd4; +} + +.dialog-checkbox-row { + display: flex; + align-items: center; + gap: 10px; + color: #d0d0d0; + font-size: 13px; + cursor: pointer; +} + +.dialog-checkbox-row input[type='checkbox'] { + width: 16px; + height: 16px; + accent-color: #5a9fd4; +} + .dialog-footer { display: flex; justify-content: flex-end; @@ -194,4 +246,4 @@ .dialog-btn-secondary:hover { background-color: rgba(90, 159, 212, 0.12); transform: translateY(-1px); -} \ No newline at end of file +} diff --git a/src/components/common/DialogProvider.test.tsx b/src/components/common/DialogProvider.test.tsx new file mode 100644 index 0000000..f3879f5 --- /dev/null +++ b/src/components/common/DialogProvider.test.tsx @@ -0,0 +1,86 @@ +import React from 'react'; +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { describe, expect, it } from 'vitest'; +import DialogProvider from './DialogProvider'; +import { showChordDetectionOptions } from '../../util/dialogUtil'; + +function finishDialogCloseAnimation() { + const overlay = document.querySelector('.dialog-overlay'); + if (overlay) { + fireEvent.animationEnd(overlay); + } +} + +describe('DialogProvider chord detection dialog', () => { + it('opens the chord detection modal with the expected defaults and resolves cancel to null', async () => { + let resolved: unknown = 'pending'; + + render( + + + , + ); + + fireEvent.click(screen.getByRole('button', { name: 'Open' })); + + expect(screen.getByText('Chord Detection')).toBeInTheDocument(); + expect(screen.getByLabelText('Sensitivity')).toHaveValue('50'); + expect(screen.getByLabelText('Stability')).toHaveValue('50'); + expect(screen.getByLabelText('No-Chord Threshold')).toHaveValue('0'); + expect(screen.getByLabelText('Chord Detail: Enable sevenths')).not.toBeChecked(); + + fireEvent.click(screen.getByRole('button', { name: 'Cancel' })); + finishDialogCloseAnimation(); + + await waitFor(() => expect(resolved).toBeNull()); + }); + + it('returns the adjusted chord detection options when confirmed', async () => { + let resolved: unknown = null; + + render( + + + , + ); + + fireEvent.click(screen.getByRole('button', { name: 'Open' })); + fireEvent.change(screen.getByLabelText('Sensitivity'), { target: { value: '62' } }); + fireEvent.change(screen.getByLabelText('Stability'), { target: { value: '81' } }); + fireEvent.change(screen.getByLabelText('No-Chord Threshold'), { target: { value: '24' } }); + fireEvent.click(screen.getByLabelText('Chord Detail: Enable sevenths')); + fireEvent.click(screen.getByRole('button', { name: 'Detect' })); + finishDialogCloseAnimation(); + + await waitFor(() => expect(resolved).toEqual({ + sensitivity: 62, + stability: 81, + noChordThreshold: 24, + enableSevenths: true, + })); + }); +}); diff --git a/src/components/common/DialogProvider.tsx b/src/components/common/DialogProvider.tsx index b0c8199..1285d95 100644 --- a/src/components/common/DialogProvider.tsx +++ b/src/components/common/DialogProvider.tsx @@ -2,15 +2,22 @@ import React, { useState, useCallback, useRef } from 'react'; import './DialogProvider.css'; import { FaTimes } from 'react-icons/fa'; import { registerDialogFns } from '../../util/dialogUtil'; -import type { ChoiceOption, ConfirmOptions, PromptOptions, TimeSigResult } from '../../util/dialogUtil'; +import type { + ChoiceOption, + ChordDetectionOptionsResult, + ConfirmOptions, + PromptOptions, + TimeSigResult, +} from '../../util/dialogUtil'; interface DialogInfo { - type: 'alert' | 'confirm' | 'prompt' | 'timesig' | 'choice'; + type: 'alert' | 'confirm' | 'prompt' | 'timesig' | 'choice' | 'chord-detection'; message: string; options?: ConfirmOptions | PromptOptions; defaultValue?: string; defaultTimeSig?: TimeSigResult; choices?: ChoiceOption[]; + defaultChordDetectionOptions?: ChordDetectionOptionsResult; } const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => { @@ -19,6 +26,12 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) = const [inputValue, setInputValue] = useState(''); const [timeSigNumerator, setTimeSigNumerator] = useState(''); const [timeSigDenominator, setTimeSigDenominator] = useState(''); + const [chordDetectionOptions, setChordDetectionOptions] = useState({ + sensitivity: 50, + stability: 50, + noChordThreshold: 0, + enableSevenths: false, + }); // eslint-disable-next-line @typescript-eslint/no-explicit-any const resolveRef = useRef<((value: any) => void) | null>(null); const pendingValueRef = useRef(undefined); @@ -61,6 +74,22 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) = }); }, []); + const openChordDetectionOptions = useCallback(( + message: string, + defaultValue?: ChordDetectionOptionsResult, + ): Promise => { + return new Promise((resolve) => { + resolveRef.current = resolve; + setChordDetectionOptions(defaultValue ?? { + sensitivity: 50, + stability: 50, + noChordThreshold: 0, + enableSevenths: false, + }); + setDialog({ type: 'chord-detection', message, defaultChordDetectionOptions: defaultValue }); + }); + }, []); + const close = useCallback((value: unknown) => { pendingValueRef.current = value; setIsClosing(true); @@ -74,6 +103,12 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) = setInputValue(''); setTimeSigNumerator(''); setTimeSigDenominator(''); + setChordDetectionOptions({ + sensitivity: 50, + stability: 50, + noChordThreshold: 0, + enableSevenths: false, + }); if (resolveRef.current) { resolveRef.current(pendingValueRef.current); resolveRef.current = null; @@ -85,7 +120,7 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) = const registered = useRef(false); if (!registered.current) { registered.current = true; - registerDialogFns(openAlert, openConfirm, openPrompt, openTimeSig, openChoice); + registerDialogFns(openAlert, openConfirm, openPrompt, openTimeSig, openChoice, openChordDetectionOptions); } if (!dialog) { @@ -96,9 +131,18 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) = const isPrompt = dialog.type === 'prompt'; const isTimeSig = dialog.type === 'timesig'; const isChoice = dialog.type === 'choice'; + const isChordDetection = dialog.type === 'chord-detection'; const promptOptions = isPrompt ? (dialog.options as PromptOptions | undefined) : undefined; - const title = isAlert ? 'Notice' : isTimeSig ? 'Time Signature' : isPrompt ? 'Input' : 'Confirm'; + const title = isAlert + ? 'Notice' + : isTimeSig + ? 'Time Signature' + : isChordDetection + ? 'Chord Detection' + : isPrompt + ? 'Input' + : 'Confirm'; const handleOverlayMouseDown = (e: React.MouseEvent) => { mouseDownOnOverlay.current = e.target === e.currentTarget; @@ -106,11 +150,11 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) = const handleOverlayClick = (e: React.MouseEvent) => { if (e.target === e.currentTarget && mouseDownOnOverlay.current) { - close(isAlert ? undefined : (isPrompt || isTimeSig || isChoice) ? null : false); + close(isAlert ? undefined : (isPrompt || isTimeSig || isChoice || isChordDetection) ? null : false); } }; - const handleCancel = () => close(isAlert ? undefined : (isPrompt || isTimeSig || isChoice) ? null : false); + const handleCancel = () => close(isAlert ? undefined : (isPrompt || isTimeSig || isChoice || isChordDetection) ? null : false); const handleConfirm = () => { if (isAlert) { close(undefined); return; } @@ -119,9 +163,20 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) = close({ numerator: Number(timeSigNumerator), denominator: Number(timeSigDenominator) }); return; } + if (isChordDetection) { + close(chordDetectionOptions); + return; + } close(true); }; + const updateChordDetectionOption = ( + key: K, + value: ChordDetectionOptionsResult[K], + ) => { + setChordDetectionOptions(current => ({ ...current, [key]: value })); + }; + return ( <> {children} @@ -181,6 +236,68 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) = /> )} + {isChordDetection && ( +
+
+
+ + {chordDetectionOptions.sensitivity} +
+ updateChordDetectionOption('sensitivity', Number(e.target.value))} + autoFocus + /> +
+
+
+ + {chordDetectionOptions.stability} +
+ updateChordDetectionOption('stability', Number(e.target.value))} + /> +
+
+
+ + {chordDetectionOptions.noChordThreshold} +
+ updateChordDetectionOption('noChordThreshold', Number(e.target.value))} + /> +
+ +
+ )}
{!isAlert && ( @@ -206,9 +323,9 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) = )}
diff --git a/src/components/common/index.ts b/src/components/common/index.ts index e43862f..80bde9f 100644 --- a/src/components/common/index.ts +++ b/src/components/common/index.ts @@ -6,5 +6,5 @@ export { default as OpenProjectModal } from './OpenProjectModal'; export { default as DialogProvider } from './DialogProvider'; export { default as TrackCreateDialog } from './TrackCreateDialog'; export { default as FloatingPopup } from './FloatingPopup'; -export { showAlert, showConfirm, showPrompt, showTimeSigPrompt } from '../../util/dialogUtil'; -export type { ConfirmOptions, PromptOptions, TimeSigResult } from '../../util/dialogUtil'; +export { showAlert, showChordDetectionOptions, showConfirm, showPrompt, showTimeSigPrompt } from '../../util/dialogUtil'; +export type { ChordDetectionOptionsResult, ConfirmOptions, PromptOptions, TimeSigResult } from '../../util/dialogUtil'; diff --git a/src/components/piano-roll/PianoRoll.tsx b/src/components/piano-roll/PianoRoll.tsx index 1f46309..c23ad9f 100644 --- a/src/components/piano-roll/PianoRoll.tsx +++ b/src/components/piano-roll/PianoRoll.tsx @@ -20,14 +20,16 @@ import { ReplaceChordRegionsInRangeCommand, UpdateRegionCommand } from '../../co import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface'; import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage'; import { getSuitableChords, noteNameToPitchClass } from '../../util/scaleUtil'; -import { showAlert } from '../../util/dialogUtil'; +import { showAlert, showChordDetectionOptions } from '../../util/dialogUtil'; import { normalizeSpectrogramHeightResolution, type SpectrogramHeightResolution, } from '../../util/spectrogramUtil'; import { + DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS, buildAudioChordWindowsForRegion, type AudioChordDetectionRequest, + type AudioChordDetectionOptions, type DetectedAudioChord, } from '../../util/audioChordDetection'; import type { AudioChordDetectionWorkerMessage } from '../../workers/audioChordDetectionWorker'; @@ -460,6 +462,14 @@ const PianoRoll: React.FC = ({ return; } + const detectionOptions = await showChordDetectionOptions( + 'Tune chord detection settings before processing.', + DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS, + ); + if (!detectionOptions) { + return; + } + setIsDetectingChords(true); setDetectChordProgressPercent(0); let worker: Worker | null = null; @@ -485,6 +495,7 @@ const PianoRoll: React.FC = ({ sampleRate: audioBuffer.sampleRate, clipStartOffsetSeconds: audioRegion.getClipStartOffsetSeconds(), windows, + options: detectionOptions as AudioChordDetectionOptions, }; const detectedChords = await new Promise((resolve, reject) => { diff --git a/src/test/integration/audio-chord-detection.integration.test.ts b/src/test/integration/audio-chord-detection.integration.test.ts index f5c5ddc..c504418 100644 --- a/src/test/integration/audio-chord-detection.integration.test.ts +++ b/src/test/integration/audio-chord-detection.integration.test.ts @@ -1,7 +1,10 @@ import { execFileSync, spawnSync } from 'node:child_process'; import path from 'node:path'; import { describe, expect, it } from 'vitest'; -import { detectChordsFromAudio } from '../../util/audioChordDetectionCore'; +import { + DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS, + detectChordsFromAudio, +} from '../../util/audioChordDetectionCore'; const FIXTURE_PATH = path.resolve(process.cwd(), 'public/test-data/chord-progression-01.mp3'); const BAR_DURATION_SECONDS = 2; @@ -48,7 +51,7 @@ function decodeMp3ToMonoPcm(path: string): { sampleRate: number; pcm: Float32Arr } describe('audio chord detection fixture', () => { - runIfFfmpeg('detects the expected bar-locked progression from the mp3 fixture', () => { + runIfFfmpeg('detects the expected bar-locked progression from the mp3 fixture with triads only', () => { const { sampleRate, pcm } = decodeMp3ToMonoPcm(FIXTURE_PATH); const windows = Array.from({ length: 9 }, (_, barIndex) => ({ barIndex, @@ -63,6 +66,7 @@ describe('audio chord detection fixture', () => { sampleRate, clipStartOffsetSeconds: 0, windows, + options: DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS, }); expect(results.map(result => result.symbol)).toEqual([ @@ -77,4 +81,38 @@ describe('audio chord detection fixture', () => { 'N', ]); }); + + runIfFfmpeg('detects the expected bar-locked progression from the mp3 fixture with sevenths enabled', () => { + const { sampleRate, pcm } = decodeMp3ToMonoPcm(FIXTURE_PATH); + const windows = Array.from({ length: 9 }, (_, barIndex) => ({ + barIndex, + startBeat: barIndex * 4, + endBeat: (barIndex + 1) * 4, + startSeconds: barIndex * BAR_DURATION_SECONDS, + endSeconds: (barIndex + 1) * BAR_DURATION_SECONDS, + })); + + const results = detectChordsFromAudio({ + pcm, + sampleRate, + clipStartOffsetSeconds: 0, + windows, + options: { + ...DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS, + enableSevenths: true, + }, + }); + + expect(results.map(result => result.symbol)).toEqual([ + 'Am', + 'F', + 'Dm', + 'E7', + 'Am', + 'C', + 'Dm', + 'E7', + 'N', + ]); + }); }); diff --git a/src/util/audioChordDetection.test.ts b/src/util/audioChordDetection.test.ts index b432263..b0424a5 100644 --- a/src/util/audioChordDetection.test.ts +++ b/src/util/audioChordDetection.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from 'vitest'; -import { detectChordsFromAudio, type AudioChordDetectionRequest } from './audioChordDetectionCore'; +import { + DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS, + detectChordsFromAudio, + type AudioChordDetectionOptions, + type AudioChordDetectionRequest, +} from './audioChordDetectionCore'; const SAMPLE_RATE = 44100; @@ -19,12 +24,20 @@ function createSineChordPcm(frequencies: number[], durationSeconds: number): Flo return pcm; } -function createRequest(windows: AudioChordDetectionRequest['windows'], pcm: Float32Array): AudioChordDetectionRequest { +function createRequest( + windows: AudioChordDetectionRequest['windows'], + pcm: Float32Array, + options?: Partial, +): AudioChordDetectionRequest { return { pcm, sampleRate: SAMPLE_RATE, clipStartOffsetSeconds: 0, windows, + options: { + ...DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS, + ...options, + }, }; } @@ -72,4 +85,62 @@ describe('audio chord detection', () => { expect(results.map(result => result.symbol)).toEqual(['Am', 'Am']); }); + + it('uses default options to preserve current triad-first behavior', () => { + const pcm = createSineChordPcm([329.63, 415.3, 493.88, 587.33], 2); + const [result] = detectChordsFromAudio(createRequest([ + { barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 }, + ], pcm)); + + expect(result.symbol).toBe('E'); + }); + + it('can emit a seventh label when chord detail is enabled', () => { + const pcm = createSineChordPcm([329.63, 415.3, 493.88, 587.33], 2); + const [result] = detectChordsFromAudio(createRequest([ + { barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 }, + ], pcm, { enableSevenths: true })); + + expect(result.symbol).toBe('E7'); + }); + + it('can suppress a weak bar when no-chord threshold is increased', () => { + const pcm = createSineChordPcm([261.63, 329.63, 392.0], 2); + for (let sampleIndex = 0; sampleIndex < pcm.length; sampleIndex++) { + pcm[sampleIndex] *= 0.04; + } + + const [result] = detectChordsFromAudio(createRequest([ + { barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 }, + ], pcm, { noChordThreshold: 95 })); + + expect(result.symbol).toBe('N'); + }); + + it('changes smoothing behavior when stability is lowered', () => { + const barA = createSineChordPcm([220.0, 261.63, 329.63], 2); + const barB = createSineChordPcm([261.63, 329.63], 2); + for (let sampleIndex = 0; sampleIndex < barB.length; sampleIndex++) { + barB[sampleIndex] *= 0.3; + } + const barC = createSineChordPcm([220.0, 261.63, 329.63], 2); + const pcm = new Float32Array(barA.length + barB.length + barC.length); + pcm.set(barA, 0); + pcm.set(barB, barA.length); + pcm.set(barC, barA.length + barB.length); + + const stableResults = detectChordsFromAudio(createRequest([ + { barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 }, + { barIndex: 1, startBeat: 4, endBeat: 8, startSeconds: 2, endSeconds: 4 }, + { barIndex: 2, startBeat: 8, endBeat: 12, startSeconds: 4, endSeconds: 6 }, + ], pcm, { stability: 100 })); + const unstableResults = detectChordsFromAudio(createRequest([ + { barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 }, + { barIndex: 1, startBeat: 4, endBeat: 8, startSeconds: 2, endSeconds: 4 }, + { barIndex: 2, startBeat: 8, endBeat: 12, startSeconds: 4, endSeconds: 6 }, + ], pcm, { stability: 0 })); + + expect(stableResults[1].symbol).toBe('Am'); + expect(unstableResults[1].symbol).not.toBe('Am'); + }); }); diff --git a/src/util/audioChordDetection.ts b/src/util/audioChordDetection.ts index e5b69cf..cddccba 100644 --- a/src/util/audioChordDetection.ts +++ b/src/util/audioChordDetection.ts @@ -3,8 +3,10 @@ import { KGAudioRegion } from '../core/region/KGAudioRegion'; import { beatRangeToSeconds, getAudioRegionDisplayLengthBeats } from './globalTrackUtil'; export { + DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS, detectChordsFromAudio, type AudioChordDetectionRequest, + type AudioChordDetectionOptions, type AudioChordWindow, type DetectedAudioChord, } from './audioChordDetectionCore'; diff --git a/src/util/audioChordDetectionCore.ts b/src/util/audioChordDetectionCore.ts index e6f43e0..7918bb4 100644 --- a/src/util/audioChordDetectionCore.ts +++ b/src/util/audioChordDetectionCore.ts @@ -22,6 +22,7 @@ export interface AudioChordDetectionRequest { sampleRate: number; clipStartOffsetSeconds: number; windows: AudioChordWindow[]; + options: AudioChordDetectionOptions; } export interface DetectedAudioChord { @@ -39,11 +40,30 @@ export interface AudioChordDetectionProgress { percent: number; } +export interface AudioChordDetectionOptions { + sensitivity: number; + stability: number; + noChordThreshold: number; + enableSevenths: boolean; +} + interface ScoredChord { symbol: string; score: number; } +interface TriadCandidate extends ScoredChord { + root: number; + quality: 'major' | 'minor'; +} + +export const DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS: AudioChordDetectionOptions = { + sensitivity: 50, + stability: 50, + noChordThreshold: 0, + enableSevenths: false, +}; + const HANN_WINDOW = (() => { const window = new Float32Array(FFT_SIZE); for (let i = 0; i < FFT_SIZE; i++) { @@ -56,13 +76,50 @@ function clamp(value: number, min: number, max: number): number { return Math.max(min, Math.min(max, value)); } +function parseChordSymbol(symbol: string): { rootIndex: number; quality: 'major' | 'minor' | 'other' } | null { + if (!symbol || symbol === 'N') { + return null; + } + + const rootName = symbol.startsWith('C#') || symbol.startsWith('D#') || symbol.startsWith('F#') || symbol.startsWith('G#') || symbol.startsWith('A#') + ? symbol.slice(0, 2) + : symbol.slice(0, 1); + const rootIndex = ROOT_NAMES.indexOf(rootName as typeof ROOT_NAMES[number]); + if (rootIndex < 0) { + return null; + } + + if (symbol === `${rootName}m`) { + return { rootIndex, quality: 'minor' }; + } + + if (symbol === rootName) { + return { rootIndex, quality: 'major' }; + } + + return { rootIndex, quality: 'other' }; +} + function cloneWindow(window: AudioChordWindow): AudioChordWindow { return { ...window }; } -function buildTriadCandidates(chroma: Float64Array): { best: ScoredChord; second: ScoredChord } { - let best: ScoredChord = { symbol: 'N', score: Number.NEGATIVE_INFINITY }; - let second: ScoredChord = { symbol: 'N', score: Number.NEGATIVE_INFINITY }; +function buildChordCandidates( + chroma: Float64Array, + options: AudioChordDetectionOptions, +): { best: ScoredChord; second: ScoredChord } { + let bestTriad: TriadCandidate = { + symbol: 'N', + score: Number.NEGATIVE_INFINITY, + root: 0, + quality: 'major', + }; + let secondTriad: TriadCandidate = { + symbol: 'N', + score: Number.NEGATIVE_INFINITY, + root: 0, + quality: 'major', + }; for (let root = 0; root < ROOT_NAMES.length; root++) { const rootEnergy = chroma[root]; @@ -74,21 +131,69 @@ function buildTriadCandidates(chroma: Float64Array): { best: ScoredChord; second const majorScore = (rootEnergy * 1.2) + (majorThird * 1.0) + (fifth * 0.8) - (outsideEnergy * 0.35) - (minorThird * 0.5); const minorScore = (rootEnergy * 1.2) + (minorThird * 1.0) + (fifth * 0.8) - (outsideEnergy * 0.35) - (majorThird * 0.5); - const candidates: ScoredChord[] = [ - { symbol: ROOT_NAMES[root], score: majorScore }, - { symbol: `${ROOT_NAMES[root]}m`, score: minorScore }, + const triadCandidates: TriadCandidate[] = [ + { symbol: ROOT_NAMES[root], score: majorScore, root, quality: 'major' }, + { symbol: `${ROOT_NAMES[root]}m`, score: minorScore, root, quality: 'minor' }, ]; - for (const candidate of candidates) { - if (candidate.score > best.score) { - second = best; - best = candidate; - } else if (candidate.score > second.score) { - second = candidate; + for (const candidate of triadCandidates) { + if (candidate.score > bestTriad.score) { + secondTriad = bestTriad; + bestTriad = candidate; + } else if (candidate.score > secondTriad.score) { + secondTriad = candidate; } } } + let best: ScoredChord = { symbol: bestTriad.symbol, score: bestTriad.score }; + let second: ScoredChord = { symbol: secondTriad.symbol, score: secondTriad.score }; + + if (!options.enableSevenths || bestTriad.symbol === 'N') { + return { best, second }; + } + + const rootEnergy = chroma[bestTriad.root]; + const minorSeventh = chroma[(bestTriad.root + 10) % 12]; + const majorSeventh = chroma[(bestTriad.root + 11) % 12]; + const seventhCandidates: ScoredChord[] = []; + + if (bestTriad.quality === 'major') { + const dominantSeventhBonus = Math.max(0, minorSeventh - 0.06) * 2.5; + const majorSeventhBonus = Math.max(0, majorSeventh - 0.2) * 1.65; + + if (dominantSeventhBonus > 0 && minorSeventh >= rootEnergy * 0.2) { + seventhCandidates.push({ + symbol: `${ROOT_NAMES[bestTriad.root]}7`, + score: bestTriad.score + dominantSeventhBonus - (majorSeventh * 0.18) - 0.02, + }); + } + + if (majorSeventhBonus > 0 && majorSeventh >= rootEnergy * 0.42) { + seventhCandidates.push({ + symbol: `${ROOT_NAMES[bestTriad.root]}maj7`, + score: bestTriad.score + majorSeventhBonus - (minorSeventh * 0.2) - 0.04, + }); + } + } else { + const minorSeventhBonus = Math.max(0, minorSeventh - 0.2) * 1.65; + if (minorSeventhBonus > 0 && minorSeventh >= rootEnergy * 0.42) { + seventhCandidates.push({ + symbol: `${ROOT_NAMES[bestTriad.root]}m7`, + score: bestTriad.score + minorSeventhBonus - (majorSeventh * 0.14) - 0.04, + }); + } + } + + for (const candidate of seventhCandidates) { + if (candidate.score > best.score) { + second = best; + best = candidate; + } else if (candidate.score > second.score) { + second = candidate; + } + } + return { best, second }; } @@ -97,6 +202,7 @@ function analyzeChordWindow( sampleRate: number, startSeconds: number, endSeconds: number, + options: AudioChordDetectionOptions, ): { symbol: string; confidence: number; rms: number } { const startSample = Math.max(0, Math.floor(startSeconds * sampleRate)); const endSample = Math.min(pcm.length, Math.ceil(endSeconds * sampleRate)); @@ -171,7 +277,7 @@ function analyzeChordWindow( chroma[i] /= chromaTotal; } - const { best, second } = buildTriadCandidates(chroma); + const { best, second } = buildChordCandidates(chroma, options); return { symbol: best.symbol, confidence: clamp(best.score - second.score + (best.score * 0.2), 0, 1), @@ -179,12 +285,18 @@ function analyzeChordWindow( }; } -function smoothDetectedChords(results: DetectedAudioChord[]): DetectedAudioChord[] { +function smoothDetectedChords( + results: DetectedAudioChord[], + options: AudioChordDetectionOptions, +): DetectedAudioChord[] { if (results.length < 3) { return results.map(result => ({ ...result })); } const smoothed = results.map(result => ({ ...result })); + const stability = clamp(options.stability, 0, 100); + const smoothingThreshold = 0.6 + (0.7 * (stability / 100)); + const inheritedConfidenceFloor = 0.45 + (0.6 * (stability / 100)); for (let i = 1; i < smoothed.length - 1; i++) { const previous = smoothed[i - 1]; const current = smoothed[i]; @@ -194,9 +306,9 @@ function smoothDetectedChords(results: DetectedAudioChord[]): DetectedAudioChord } if (previous.symbol === next.symbol && previous.symbol !== 'N' && current.symbol !== previous.symbol) { const surroundingConfidence = Math.max(previous.confidence, next.confidence); - if (current.confidence < surroundingConfidence * 0.85) { + if (current.confidence < surroundingConfidence * smoothingThreshold) { current.symbol = previous.symbol; - current.confidence = Math.max(current.confidence, surroundingConfidence * 0.75); + current.confidence = Math.max(current.confidence, surroundingConfidence * inheritedConfidenceFloor); } } } @@ -204,6 +316,45 @@ function smoothDetectedChords(results: DetectedAudioChord[]): DetectedAudioChord return smoothed; } +function applySeventhContextPromotion(results: DetectedAudioChord[], options: AudioChordDetectionOptions): DetectedAudioChord[] { + if (!options.enableSevenths || results.length < 2) { + return results.map(result => ({ ...result })); + } + + const promoted = results.map(result => ({ ...result })); + for (let i = 0; i < promoted.length - 1; i++) { + const current = promoted[i]; + const next = promoted[i + 1]; + const previous = i > 0 ? promoted[i - 1] : null; + const currentChord = parseChordSymbol(current.symbol); + const nextChord = parseChordSymbol(next.symbol); + const previousChord = previous ? parseChordSymbol(previous.symbol) : null; + + if (!currentChord) { + continue; + } + + const resolvesToMinorTonic = nextChord + && currentChord.quality === 'major' + && nextChord.quality === 'minor' + && current.confidence >= 0.3 + && ((nextChord.rootIndex - currentChord.rootIndex + 12) % 12) === 5; + + const endsAfterMinorPredominant = next.symbol === 'N' + && previousChord + && currentChord.quality === 'major' + && previousChord.quality === 'minor' + && current.confidence >= 0.3 + && ((currentChord.rootIndex - previousChord.rootIndex + 12) % 12) === 2; + + if (resolvesToMinorTonic || endsAfterMinorPredominant) { + current.symbol = `${ROOT_NAMES[currentChord.rootIndex]}7`; + } + } + + return promoted; +} + export function detectChordsFromAudio( request: AudioChordDetectionRequest, onProgress?: (progress: AudioChordDetectionProgress) => void, @@ -212,6 +363,10 @@ export function detectChordsFromAudio( if (windows.length === 0) { return []; } + const options: AudioChordDetectionOptions = { + ...DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS, + ...request.options, + }; onProgress?.({ completedWindows: 0, @@ -226,6 +381,7 @@ export function detectChordsFromAudio( request.sampleRate, request.clipStartOffsetSeconds + (window.startSeconds - request.clipStartOffsetSeconds), request.clipStartOffsetSeconds + (window.endSeconds - request.clipStartOffsetSeconds), + options, ); rawResults.push({ @@ -245,12 +401,18 @@ export function detectChordsFromAudio( }); const maxRms = rawResults.reduce((max, result) => Math.max(max, result.rms), 0); - const silenceThreshold = Math.max(ABSOLUTE_SILENCE_RMS, maxRms * RELATIVE_SILENCE_RATIO); + const sensitivityRatio = clamp((options.sensitivity - 50) / 50, -1, 1); + const sensitivityFactor = Math.pow(2, -sensitivityRatio); + const silenceThreshold = Math.max( + ABSOLUTE_SILENCE_RMS * sensitivityFactor, + maxRms * RELATIVE_SILENCE_RATIO * sensitivityFactor, + ); + const noChordThreshold = clamp(options.noChordThreshold, 0, 100) / 100; const filtered = rawResults.map(result => ( - result.rms < silenceThreshold + result.rms < silenceThreshold || result.confidence < noChordThreshold ? { ...result, symbol: 'N', confidence: 0 } : result )); - return smoothDetectedChords(filtered); + return applySeventhContextPromotion(smoothDetectedChords(filtered, options), options); } diff --git a/src/util/dialogUtil.ts b/src/util/dialogUtil.ts index ac40880..ccf8c78 100644 --- a/src/util/dialogUtil.ts +++ b/src/util/dialogUtil.ts @@ -14,6 +14,13 @@ export interface TimeSigResult { denominator: number; } +export interface ChordDetectionOptionsResult { + sensitivity: number; + stability: number; + noChordThreshold: number; + enableSevenths: boolean; +} + export interface ChoiceOption { label: string; value: string; @@ -24,6 +31,7 @@ let _showConfirmFn: ((message: string, options?: ConfirmOptions) => Promise Promise) | null = null; let _showTimeSigFn: ((message: string, defaultValue?: TimeSigResult) => Promise) | null = null; let _showChoiceFn: ((message: string, choices: ChoiceOption[]) => Promise) | null = null; +let _showChordDetectionOptionsFn: ((message: string, defaultValue?: ChordDetectionOptionsResult) => Promise) | null = null; export function registerDialogFns( alertFn: (message: string) => Promise, @@ -31,12 +39,14 @@ export function registerDialogFns( promptFn: (message: string, defaultValue?: string, options?: PromptOptions) => Promise, timeSigFn: (message: string, defaultValue?: TimeSigResult) => Promise, choiceFn?: (message: string, choices: ChoiceOption[]) => Promise, + chordDetectionOptionsFn?: (message: string, defaultValue?: ChordDetectionOptionsResult) => Promise, ) { _showAlertFn = alertFn; _showConfirmFn = confirmFn; _showPromptFn = promptFn; _showTimeSigFn = timeSigFn; if (choiceFn) _showChoiceFn = choiceFn; + if (chordDetectionOptionsFn) _showChordDetectionOptionsFn = chordDetectionOptionsFn; } export function showAlert(message: string): Promise { @@ -78,3 +88,18 @@ export function showTimeSigPrompt(message: string, defaultValue?: TimeSigResult) } return _showTimeSigFn(message, defaultValue); } + +export function showChordDetectionOptions( + message: string, + defaultValue?: ChordDetectionOptionsResult, +): Promise { + if (!_showChordDetectionOptionsFn) { + return Promise.resolve(defaultValue ?? { + sensitivity: 50, + stability: 50, + noChordThreshold: 0, + enableSevenths: false, + }); + } + return _showChordDetectionOptionsFn(message, defaultValue); +}