diff --git a/public/test-data/chord-progression-01.mp3 b/public/test-data/chord-progression-01.mp3 new file mode 100644 index 0000000..d313925 Binary files /dev/null and b/public/test-data/chord-progression-01.mp3 differ diff --git a/src/components/piano-roll/PianoRoll.css b/src/components/piano-roll/PianoRoll.css index e220217..935a596 100644 --- a/src/components/piano-roll/PianoRoll.css +++ b/src/components/piano-roll/PianoRoll.css @@ -235,6 +235,11 @@ left: 0; } +.piano-roll-toolbar .quant-option.disabled { + color: #666; + cursor: default; +} + .piano-roll-automation-toolbar-group { display: flex; align-items: center; @@ -408,11 +413,37 @@ } .spectrogram-loading-label { - padding: 6px 10px; - background: rgba(0, 0, 0, 0.6); - color: #aaa; + min-width: 240px; + padding: 10px 12px; + background: rgba(0, 0, 0, 0.72); + color: #ddd; font-size: 11px; - border-radius: 3px; + border: 1px solid #3a3a3a; + border-radius: 6px; + display: flex; + flex-direction: column; + gap: 8px; +} + +.piano-roll-progress-block { + display: flex; + flex-direction: column; + gap: 6px; +} + +.piano-roll-progress-track { + width: 100%; + height: 8px; + background-color: #1d1d1d; + border: 1px solid #3a3a3a; + border-radius: 999px; + overflow: hidden; +} + +.piano-roll-progress-fill { + height: 100%; + background: linear-gradient(90deg, #5a9fd4 0%, #7cc2f1 100%); + transition: width 0.15s linear; } .piano-grid-header { diff --git a/src/components/piano-roll/PianoRoll.tsx b/src/components/piano-roll/PianoRoll.tsx index 0da1aee..1f46309 100644 --- a/src/components/piano-roll/PianoRoll.tsx +++ b/src/components/piano-roll/PianoRoll.tsx @@ -1,5 +1,6 @@ import React, { useRef, useEffect, useState, useCallback, useLayoutEffect, useMemo } from 'react'; import './PianoRoll.css'; +import * as Tone from 'tone'; import { useProjectStore } from '../../stores/projectStore'; import { FaGripLines } from 'react-icons/fa'; import { KGMidiRegion } from '../../core/region/KGMidiRegion'; @@ -15,13 +16,21 @@ import { KGMidiTrack, type InstrumentType } from '../../core/track/KGMidiTrack'; import { KGPianoRollState } from '../../core/state/KGPianoRollState'; import { ConfigManager } from '../../core/config/ConfigManager'; import { beatsToBar } from '../../util/midiUtil'; -import { UpdateRegionCommand } from '../../core/commands'; +import { ReplaceChordRegionsInRangeCommand, UpdateRegionCommand } from '../../core/commands'; +import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface'; +import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage'; import { getSuitableChords, noteNameToPitchClass } from '../../util/scaleUtil'; import { showAlert } from '../../util/dialogUtil'; import { normalizeSpectrogramHeightResolution, type SpectrogramHeightResolution, } from '../../util/spectrogramUtil'; +import { + buildAudioChordWindowsForRegion, + type AudioChordDetectionRequest, + type DetectedAudioChord, +} from '../../util/audioChordDetection'; +import type { AudioChordDetectionWorkerMessage } from '../../workers/audioChordDetectionWorker'; import type { PianoRollAutomationType } from './pianoRollAutomation'; import type { SheetMeasureMetric } from './sheetNotationTypes'; import { getSheetPlayheadPixel, getSheetQuantizationOptions, parseSheetQuantization } from './sheetNotation'; @@ -59,7 +68,7 @@ const PianoRoll: React.FC = ({ }) => { const isSpectrogram = mode === 'spectrogram'; const isHybrid = mode === 'hybrid'; - const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showKGOnePanel, showEventListPanel, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled, bpm, pianoRollScrollRequest, selectedNoteIds, automationRedrawVersion } = useProjectStore(); + const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showKGOnePanel, showEventListPanel, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled, bpm, pianoRollScrollRequest, selectedNoteIds, automationRedrawVersion, refreshProjectState } = useProjectStore(); // Tool state for piano roll const [activeTool, setActiveTool] = useState<'pointer' | 'pencil'>('pointer'); @@ -69,6 +78,8 @@ const PianoRoll: React.FC = ({ const [spectrogramPower, setSpectrogramPower] = useState(0.5); const [spectrogramHeightResolution, setSpectrogramHeightResolution] = useState(3); + const [isDetectingChords, setIsDetectingChords] = useState(false); + const [detectChordProgressPercent, setDetectChordProgressPercent] = useState(0); // Piano roll zoom (1x–8x); updates --region-grid-beat-width CSS variable const [pianoRollZoom, setPianoRollZoom] = useState(() => KGPianoRollState.instance().getPianoRollZoom()); @@ -436,6 +447,92 @@ const PianoRoll: React.FC = ({ } }; + const handleDetectChords = useCallback(async () => { + if (!audioRegion || !projectName || !trackId) { + await showAlert('Open an audio region in spectrogram mode before detecting chords.'); + return; + } + + const project = KGCore.instance().getCurrentProject(); + const windows = buildAudioChordWindowsForRegion(project, audioRegion); + if (windows.length === 0) { + await showAlert('The selected audio region has no audible span to analyze.'); + return; + } + + setIsDetectingChords(true); + setDetectChordProgressPercent(0); + let worker: Worker | null = null; + + try { + let audioBuffer = KGAudioInterface.instance().getAudioBuffer(trackId, audioRegion.getAudioFileId()); + if (!audioBuffer) { + const rawBuffer = await KGAudioFileStorage.loadAudioFile(projectName, audioRegion.getAudioFileId()); + const actx = Tone.getContext().rawContext as AudioContext; + audioBuffer = await actx.decodeAudioData(rawBuffer); + } + + const monoPcm = new Float32Array(audioBuffer.length); + for (let channelIndex = 0; channelIndex < audioBuffer.numberOfChannels; channelIndex++) { + const channelData = audioBuffer.getChannelData(channelIndex); + for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) { + monoPcm[sampleIndex] += channelData[sampleIndex] / audioBuffer.numberOfChannels; + } + } + + const request: AudioChordDetectionRequest = { + pcm: monoPcm, + sampleRate: audioBuffer.sampleRate, + clipStartOffsetSeconds: audioRegion.getClipStartOffsetSeconds(), + windows, + }; + + const detectedChords = await new Promise((resolve, reject) => { + worker = new Worker( + new URL('../../workers/audioChordDetectionWorker.ts', import.meta.url), + { type: 'module' }, + ); + + worker.onmessage = (event: MessageEvent) => { + if (event.data.type === 'progress') { + setDetectChordProgressPercent(event.data.progress.percent); + return; + } + + resolve(event.data.results); + }; + worker.onerror = () => { + reject(new Error('Chord detection worker failed.')); + }; + worker.postMessage(request, [request.pcm.buffer]); + }); + + const replacements = detectedChords + .filter(result => result.symbol !== 'N' && result.endBeat > result.startBeat) + .map(result => ({ + startBeat: result.startBeat, + length: result.endBeat - result.startBeat, + symbol: result.symbol, + })); + + const spanStartBeat = windows[0].startBeat; + const spanEndBeat = windows[windows.length - 1].endBeat; + KGCore.instance().executeCommand( + new ReplaceChordRegionsInRangeCommand(spanStartBeat, spanEndBeat, replacements), + ); + refreshProjectState(); + } catch (error) { + console.error('Error detecting chords:', error); + await showAlert('Failed to detect chords from this audio region.'); + } finally { + if (worker) { + worker.terminate(); + } + setIsDetectingChords(false); + setDetectChordProgressPercent(0); + } + }, [audioRegion, projectName, refreshProjectState, trackId]); + // Handle title click to rename the region const handleTitleClick = () => { // If we were just dragging, don't show the rename dialog @@ -1315,6 +1412,8 @@ const PianoRoll: React.FC = ({ automationType={automationType} onAutomationToggle={handleAutomationToggle} onAutomationTypeChange={handleAutomationTypeChange} + onDetectChords={handleDetectChords} + detectingChords={isDetectingChords} /> @@ -1351,6 +1450,8 @@ const PianoRoll: React.FC = ({ sheetKeySignature={keySignature} sheetInstrument={activeInstrument} onSheetMeasureMetricsChange={handleSheetMeasureMetricsChange} + overlayMessage={isDetectingChords ? `Detecting chords… (${detectChordProgressPercent.toString().padStart(2, '0')}% completed)` : null} + overlayProgressPercent={isDetectingChords ? detectChordProgressPercent : null} />
{ expect(screen.queryByTestId('automation-lane')).not.toBeInTheDocument(); }); + it('shows an overlay message when one is supplied by the parent panel', () => { + render( + + ); + + expect(screen.getByText('Detecting chords…')).toBeInTheDocument(); + expect(screen.getByRole('progressbar')).toHaveAttribute('aria-valuenow', '42'); + }); + it('renders sheet mode without piano keys or automation lane', () => { render( void; + overlayMessage?: string | null; + overlayProgressPercent?: number | null; } const PianoRollContent: React.FC = ({ @@ -89,6 +91,8 @@ const PianoRollContent: React.FC = ({ sheetKeySignature = 'C major', sheetInstrument = 'acoustic_grand_piano', onSheetMeasureMetricsChange, + overlayMessage = null, + overlayProgressPercent = null, }) => { const isSpectrogram = mode === 'spectrogram'; const showAutomationLane = automationEnabled && !isSpectrogram && !sheetMusicViewEnabled; @@ -321,6 +325,8 @@ const PianoRollContent: React.FC = ({ container.scrollTop = 0; }, [noteScrollRef, sheetMusicViewEnabled]); + const effectiveOverlayMessage = overlayMessage ?? (spectrogramLoading ? 'Computing spectrogram…' : null); + return (
= ({
)}
- {spectrogramLoading && ( + {effectiveOverlayMessage && (
- Computing spectrogram… + {effectiveOverlayMessage} + {overlayProgressPercent !== null && ( +
+
+
+
+
+ )}
)} diff --git a/src/components/piano-roll/PianoRollToolbar.test.tsx b/src/components/piano-roll/PianoRollToolbar.test.tsx index 7e50a4f..935245f 100644 --- a/src/components/piano-roll/PianoRollToolbar.test.tsx +++ b/src/components/piano-roll/PianoRollToolbar.test.tsx @@ -60,6 +60,7 @@ describe('PianoRollToolbar', () => { onChordGuideChange: vi.fn(), zoom: 1, onZoomChange: vi.fn(), + onDetectChords: vi.fn(), }; it('shows automation controls in midi mode and toggles the lane', () => { @@ -118,6 +119,44 @@ describe('PianoRollToolbar', () => { expect(screen.queryByRole('button', { name: /Pitch Bend/i })).not.toBeInTheDocument(); }); + it('shows the detect chords action in spectrogram mode and triggers it', () => { + const onDetectChords = vi.fn(); + + render( + + ); + + fireEvent.click(screen.getByTitle('More options')); + fireEvent.click(screen.getByText('Detect chords...')); + + expect(onDetectChords).toHaveBeenCalledTimes(1); + }); + + it('disables the detect chords action while detection is running', () => { + const onDetectChords = vi.fn(); + + render( + + ); + + fireEvent.click(screen.getByTitle('More options')); + const detectItem = screen.getByText('Detecting chords...'); + expect(detectItem).toHaveAttribute('aria-disabled', 'true'); + fireEvent.click(detectItem); + expect(onDetectChords).not.toHaveBeenCalled(); + }); + it('shows only the sheet controls when sheet mode is enabled', () => { render( void; onAutomationTypeChange?: (value: PianoRollAutomationType) => void; + onDetectChords?: () => void | Promise; + detectingChords?: boolean; } const PianoRollToolbar: React.FC = ({ @@ -82,6 +84,8 @@ const PianoRollToolbar: React.FC = ({ automationType = 'pitch-bend', onAutomationToggle, onAutomationTypeChange, + onDetectChords, + detectingChords = false, }) => { const showMidiControls = mode !== 'spectrogram' && !sheetMusicViewEnabled; // midi-edit and hybrid const showSpecControls = !sheetMusicViewEnabled && (mode === 'spectrogram' || mode === 'hybrid'); @@ -100,6 +104,20 @@ const PianoRollToolbar: React.FC = ({ return () => document.removeEventListener('mousedown', handleClickOutside); }, [showZoomSlider]); + const [showSpecMenu, setShowSpecMenu] = React.useState(false); + const specMenuRef = React.useRef(null); + + React.useEffect(() => { + if (!showSpecMenu) return; + const handleClickOutside = (e: MouseEvent) => { + if (specMenuRef.current && !specMenuRef.current.contains(e.target as Node)) { + setShowSpecMenu(false); + } + }; + document.addEventListener('mousedown', handleClickOutside); + return () => document.removeEventListener('mousedown', handleClickOutside); + }, [showSpecMenu]); + return (
{showMidiControls && ( @@ -280,6 +298,35 @@ const PianoRollToolbar: React.FC = ({ )}
)} + + {showSpecControls && ( +
+ + {showSpecMenu && ( +
+
{ + if (!onDetectChords || detectingChords) { + return; + } + setShowSpecMenu(false); + void onDetectChords(); + }} + aria-disabled={!onDetectChords || detectingChords} + > + {detectingChords ? 'Detecting chords...' : 'Detect chords...'} +
+
+ )} +
+ )}
); diff --git a/src/core/commands/global-region/GlobalChordCommands.test.ts b/src/core/commands/global-region/GlobalChordCommands.test.ts index b03480d..7f57ecf 100644 --- a/src/core/commands/global-region/GlobalChordCommands.test.ts +++ b/src/core/commands/global-region/GlobalChordCommands.test.ts @@ -6,6 +6,7 @@ import { KGChordRegion } from '../../region/KGChordRegion'; import { CreateChordRegionCommand } from './CreateChordRegionCommand'; import { InsertChordRegionAtBeatCommand } from './InsertChordRegionAtBeatCommand'; import { MoveGlobalRegionCommand } from './MoveGlobalRegionCommand'; +import { ReplaceChordRegionsInRangeCommand } from './ReplaceChordRegionsInRangeCommand'; import { ResizeGlobalRegionCommand } from './ResizeGlobalRegionCommand'; import { UpdateChordRegionCommand } from './UpdateChordRegionCommand'; @@ -104,4 +105,46 @@ describe('global chord region commands', () => { expect(chordTrack.getRegions()).toHaveLength(1); expect(region.getLength()).toBe(8); }); + + it('replaces only the requested chord span and restores the original layout on undo', () => { + const chordTrack = getChordTrack(); + chordTrack.setRegions([ + new KGChordRegion('left', chordTrack.getId(), chordTrack.getTrackIndex(), 'C', 0, 4), + new KGChordRegion('middle', chordTrack.getId(), chordTrack.getTrackIndex(), 'Am', 4, 4), + new KGChordRegion('right', chordTrack.getId(), chordTrack.getTrackIndex(), 'F', 8, 4), + ]); + + const command = new ReplaceChordRegionsInRangeCommand(2, 10, [ + { startBeat: 2, length: 2, symbol: 'Dm' }, + { startBeat: 4, length: 4, symbol: 'E' }, + { startBeat: 8, length: 2, symbol: 'Am' }, + ]); + + command.execute(); + + const replacedRegions = getChordTrack().getRegions() as KGChordRegion[]; + expect(replacedRegions.map(region => ({ + symbol: region.getSymbol(), + start: region.getStartFromBeat(), + length: region.getLength(), + }))).toEqual([ + { symbol: 'C', start: 0, length: 2 }, + { symbol: 'Dm', start: 2, length: 2 }, + { symbol: 'E', start: 4, length: 4 }, + { symbol: 'Am', start: 8, length: 2 }, + { symbol: 'F', start: 10, length: 2 }, + ]); + + command.undo(); + const restoredRegions = getChordTrack().getRegions() as KGChordRegion[]; + expect(restoredRegions.map(region => ({ + symbol: region.getSymbol(), + start: region.getStartFromBeat(), + length: region.getLength(), + }))).toEqual([ + { symbol: 'C', start: 0, length: 4 }, + { symbol: 'Am', start: 4, length: 4 }, + { symbol: 'F', start: 8, length: 4 }, + ]); + }); }); diff --git a/src/core/commands/global-region/ReplaceChordRegionsInRangeCommand.ts b/src/core/commands/global-region/ReplaceChordRegionsInRangeCommand.ts new file mode 100644 index 0000000..bf0a05c --- /dev/null +++ b/src/core/commands/global-region/ReplaceChordRegionsInRangeCommand.ts @@ -0,0 +1,132 @@ +import { KGCommand } from '../KGCommand'; +import { KGCore } from '../../KGCore'; +import { GlobalTrackType } from '../../global-track'; +import { KGChordRegion } from '../../region/KGChordRegion'; +import { findGlobalTrackByType } from '../../../util/globalTrackUtil'; +import { generateUniqueId } from '../../../util/miscUtil'; + +export interface ChordRegionReplacementData { + startBeat: number; + length: number; + symbol: string; +} + +function cloneChordRegion(region: KGChordRegion): KGChordRegion { + return new KGChordRegion( + region.getId(), + region.getTrackId(), + region.getTrackIndex(), + region.getSymbol(), + region.getStartFromBeat(), + region.getLength(), + ); +} + +function cloneChordRegions(regions: KGChordRegion[]): KGChordRegion[] { + return regions.map(cloneChordRegion); +} + +export class ReplaceChordRegionsInRangeCommand extends KGCommand { + private readonly rangeStartBeat: number; + private readonly rangeEndBeat: number; + private readonly replacements: ChordRegionReplacementData[]; + private originalRegions: KGChordRegion[] | null = null; + private nextRegions: KGChordRegion[] | null = null; + + constructor(rangeStartBeat: number, rangeEndBeat: number, replacements: ChordRegionReplacementData[]) { + super(); + this.rangeStartBeat = Math.max(0, rangeStartBeat); + this.rangeEndBeat = Math.max(this.rangeStartBeat, rangeEndBeat); + this.replacements = replacements.map(replacement => ({ + startBeat: replacement.startBeat, + length: replacement.length, + symbol: replacement.symbol, + })); + } + + execute(): void { + const project = KGCore.instance().getCurrentProject(); + const chordTrack = findGlobalTrackByType(project, GlobalTrackType.Chord); + if (!chordTrack) { + throw new Error('Chord global track not found'); + } + + if (this.nextRegions) { + chordTrack.setRegions(cloneChordRegions(this.nextRegions)); + return; + } + + const currentRegions = chordTrack.getRegions() + .filter((region): region is KGChordRegion => region instanceof KGChordRegion) + .sort((left, right) => left.getStartFromBeat() - right.getStartFromBeat()); + + this.originalRegions = cloneChordRegions(currentRegions); + + const preservedRegions: KGChordRegion[] = []; + for (const region of currentRegions) { + const regionStart = region.getStartFromBeat(); + const regionEnd = regionStart + region.getLength(); + + if (regionEnd <= this.rangeStartBeat || regionStart >= this.rangeEndBeat) { + preservedRegions.push(cloneChordRegion(region)); + continue; + } + + if (regionStart < this.rangeStartBeat) { + preservedRegions.push(new KGChordRegion( + region.getId(), + region.getTrackId(), + region.getTrackIndex(), + region.getSymbol(), + regionStart, + this.rangeStartBeat - regionStart, + )); + } + + if (regionEnd > this.rangeEndBeat) { + preservedRegions.push(new KGChordRegion( + generateUniqueId('KGChordRegion'), + region.getTrackId(), + region.getTrackIndex(), + region.getSymbol(), + this.rangeEndBeat, + regionEnd - this.rangeEndBeat, + )); + } + } + + const replacementRegions = this.replacements + .filter(replacement => replacement.length > 0 && replacement.symbol.trim() !== '') + .map(replacement => new KGChordRegion( + generateUniqueId('KGChordRegion'), + chordTrack.getId(), + chordTrack.getTrackIndex(), + replacement.symbol, + replacement.startBeat, + replacement.length, + )); + + this.nextRegions = [...preservedRegions, ...replacementRegions] + .sort((left, right) => left.getStartFromBeat() - right.getStartFromBeat()); + + chordTrack.setRegions(cloneChordRegions(this.nextRegions)); + } + + undo(): void { + if (!this.originalRegions) { + throw new Error('Cannot undo chord replacement without original regions'); + } + + const project = KGCore.instance().getCurrentProject(); + const chordTrack = findGlobalTrackByType(project, GlobalTrackType.Chord); + if (!chordTrack) { + throw new Error('Chord global track not found during undo'); + } + + chordTrack.setRegions(cloneChordRegions(this.originalRegions)); + } + + getDescription(): string { + return 'Replace chord regions in range'; + } +} diff --git a/src/core/commands/index.ts b/src/core/commands/index.ts index 346259d..c55011b 100644 --- a/src/core/commands/index.ts +++ b/src/core/commands/index.ts @@ -39,6 +39,10 @@ export { MergeMidiRegionsCommand } from './region/MergeMidiRegionsCommand'; export { CreateGlobalMarkerRegionCommand } from './global-region/CreateGlobalMarkerRegionCommand'; export { CreateChordRegionCommand } from './global-region/CreateChordRegionCommand'; export { InsertChordRegionAtBeatCommand } from './global-region/InsertChordRegionAtBeatCommand'; +export { + ReplaceChordRegionsInRangeCommand, + type ChordRegionReplacementData, +} from './global-region/ReplaceChordRegionsInRangeCommand'; export { CreateKeySignatureRegionCommand } from './global-region/CreateKeySignatureRegionCommand'; export { CreateTempoRegionCommand } from './global-region/CreateTempoRegionCommand'; export { MoveGlobalRegionCommand } from './global-region/MoveGlobalRegionCommand'; diff --git a/src/test/integration/audio-chord-detection.integration.test.ts b/src/test/integration/audio-chord-detection.integration.test.ts new file mode 100644 index 0000000..f5c5ddc --- /dev/null +++ b/src/test/integration/audio-chord-detection.integration.test.ts @@ -0,0 +1,80 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { detectChordsFromAudio } from '../../util/audioChordDetectionCore'; + +const FIXTURE_PATH = path.resolve(process.cwd(), 'public/test-data/chord-progression-01.mp3'); +const BAR_DURATION_SECONDS = 2; +const ffmpegAvailable = spawnSync('ffmpeg', ['-version'], { stdio: 'ignore' }).status === 0; +const runIfFfmpeg = ffmpegAvailable ? it : it.skip; + +function decodeMp3ToMonoPcm(path: string): { sampleRate: number; pcm: Float32Array } { + const wav = execFileSync( + 'ffmpeg', + ['-v', 'error', '-i', path, '-ac', '1', '-f', 'wav', 'pipe:1'], + { maxBuffer: 64 * 1024 * 1024 }, + ); + + const readString = (offset: number, length: number) => wav.toString('ascii', offset, offset + length); + if (readString(0, 4) !== 'RIFF' || readString(8, 4) !== 'WAVE') { + throw new Error('ffmpeg did not return a RIFF/WAVE stream'); + } + + let offset = 12; + let sampleRate = 44100; + let pcmData = Buffer.alloc(0); + + while (offset + 8 <= wav.length) { + const chunkId = readString(offset, 4); + const chunkSize = wav.readUInt32LE(offset + 4); + const chunkStart = offset + 8; + + if (chunkId === 'fmt ') { + sampleRate = wav.readUInt32LE(chunkStart + 4); + } else if (chunkId === 'data') { + pcmData = wav.subarray(chunkStart, chunkStart + chunkSize); + break; + } + + offset = chunkStart + chunkSize + (chunkSize % 2); + } + + const pcm = new Float32Array(pcmData.length / 2); + for (let sampleIndex = 0; sampleIndex < pcm.length; sampleIndex++) { + pcm[sampleIndex] = pcmData.readInt16LE(sampleIndex * 2) / 32768; + } + + return { sampleRate, pcm }; +} + +describe('audio chord detection fixture', () => { + runIfFfmpeg('detects the expected bar-locked progression from the mp3 fixture', () => { + const { sampleRate, pcm } = decodeMp3ToMonoPcm(FIXTURE_PATH); + const windows = Array.from({ length: 9 }, (_, barIndex) => ({ + barIndex, + startBeat: barIndex * 4, + endBeat: (barIndex + 1) * 4, + startSeconds: barIndex * BAR_DURATION_SECONDS, + endSeconds: (barIndex + 1) * BAR_DURATION_SECONDS, + })); + + const results = detectChordsFromAudio({ + pcm, + sampleRate, + clipStartOffsetSeconds: 0, + windows, + }); + + expect(results.map(result => result.symbol)).toEqual([ + 'Am', + 'F', + 'Dm', + 'E', + 'Am', + 'C', + 'Dm', + 'E', + 'N', + ]); + }); +}); diff --git a/src/util/audioChordDetection.test.ts b/src/util/audioChordDetection.test.ts new file mode 100644 index 0000000..b432263 --- /dev/null +++ b/src/util/audioChordDetection.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from 'vitest'; +import { detectChordsFromAudio, type AudioChordDetectionRequest } from './audioChordDetectionCore'; + +const SAMPLE_RATE = 44100; + +function createSineChordPcm(frequencies: number[], durationSeconds: number): Float32Array { + const sampleCount = Math.floor(durationSeconds * SAMPLE_RATE); + const pcm = new Float32Array(sampleCount); + + for (let sampleIndex = 0; sampleIndex < sampleCount; sampleIndex++) { + const time = sampleIndex / SAMPLE_RATE; + let sample = 0; + for (const frequency of frequencies) { + sample += Math.sin(2 * Math.PI * frequency * time); + } + pcm[sampleIndex] = (sample / Math.max(1, frequencies.length)) * 0.35; + } + + return pcm; +} + +function createRequest(windows: AudioChordDetectionRequest['windows'], pcm: Float32Array): AudioChordDetectionRequest { + return { + pcm, + sampleRate: SAMPLE_RATE, + clipStartOffsetSeconds: 0, + windows, + }; +} + +describe('audio chord detection', () => { + it('detects a major triad from synthetic audio', () => { + const pcm = createSineChordPcm([261.63, 329.63, 392.0], 2); + const [result] = detectChordsFromAudio(createRequest([ + { barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 }, + ], pcm)); + + expect(result.symbol).toBe('C'); + expect(result.confidence).toBeGreaterThan(0); + }); + + it('detects a minor triad from synthetic audio', () => { + const pcm = createSineChordPcm([220.0, 261.63, 329.63], 2); + const [result] = detectChordsFromAudio(createRequest([ + { barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 }, + ], pcm)); + + expect(result.symbol).toBe('Am'); + }); + + it('marks silent analysis windows as no chord', () => { + const pcm = new Float32Array(SAMPLE_RATE * 2); + const [result] = detectChordsFromAudio(createRequest([ + { barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 }, + ], pcm)); + + expect(result.symbol).toBe('N'); + expect(result.confidence).toBe(0); + }); + + it('keeps neighboring synthetic bars stable', () => { + const barA = createSineChordPcm([220.0, 261.63, 329.63], 2); + const barB = createSineChordPcm([220.0, 261.63, 329.63], 2); + const pcm = new Float32Array(barA.length + barB.length); + pcm.set(barA, 0); + pcm.set(barB, barA.length); + + const results = detectChordsFromAudio(createRequest([ + { barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 }, + { barIndex: 1, startBeat: 4, endBeat: 8, startSeconds: 2, endSeconds: 4 }, + ], pcm)); + + expect(results.map(result => result.symbol)).toEqual(['Am', 'Am']); + }); +}); diff --git a/src/util/audioChordDetection.ts b/src/util/audioChordDetection.ts new file mode 100644 index 0000000..e5b69cf --- /dev/null +++ b/src/util/audioChordDetection.ts @@ -0,0 +1,55 @@ +import { KGProject } from '../core/KGProject'; +import { KGAudioRegion } from '../core/region/KGAudioRegion'; +import { beatRangeToSeconds, getAudioRegionDisplayLengthBeats } from './globalTrackUtil'; + +export { + detectChordsFromAudio, + type AudioChordDetectionRequest, + type AudioChordWindow, + type DetectedAudioChord, +} from './audioChordDetectionCore'; + +import type { AudioChordWindow } from './audioChordDetectionCore'; + +export function buildAudioChordWindowsForRegion( + project: KGProject, + audioRegion: KGAudioRegion, +): AudioChordWindow[] { + const regionStartBeat = audioRegion.getStartFromBeat(); + const visibleLengthBeats = getAudioRegionDisplayLengthBeats(project, audioRegion); + if (visibleLengthBeats <= 0) { + return []; + } + + const regionEndBeat = regionStartBeat + visibleLengthBeats; + const beatsPerBar = project.getTimeSignature().numerator; + const startBarIndex = Math.floor(regionStartBeat / beatsPerBar); + const lastBeatExclusive = regionEndBeat - 1e-9; + const endBarIndexExclusive = Math.max( + startBarIndex + 1, + Math.ceil(Math.max(regionStartBeat, lastBeatExclusive) / beatsPerBar), + ); + + const windows: AudioChordWindow[] = []; + for (let barIndex = startBarIndex; barIndex < endBarIndexExclusive; barIndex++) { + const barStartBeat = barIndex * beatsPerBar; + const barEndBeat = barStartBeat + beatsPerBar; + const overlapStartBeat = Math.max(regionStartBeat, barStartBeat); + const overlapEndBeat = Math.min(regionEndBeat, barEndBeat); + if (overlapEndBeat <= overlapStartBeat) { + continue; + } + + const startSeconds = audioRegion.getClipStartOffsetSeconds() + beatRangeToSeconds(project, regionStartBeat, overlapStartBeat); + const endSeconds = audioRegion.getClipStartOffsetSeconds() + beatRangeToSeconds(project, regionStartBeat, overlapEndBeat); + windows.push({ + barIndex, + startBeat: overlapStartBeat, + endBeat: overlapEndBeat, + startSeconds, + endSeconds, + }); + } + + return windows; +} diff --git a/src/util/audioChordDetectionCore.ts b/src/util/audioChordDetectionCore.ts new file mode 100644 index 0000000..e6f43e0 --- /dev/null +++ b/src/util/audioChordDetectionCore.ts @@ -0,0 +1,256 @@ +import FFT from 'fft.js'; + +const FFT_SIZE = 4096; +const HOP_SIZE = 512; +const MIN_ANALYSIS_FREQUENCY = 55; +const MAX_ANALYSIS_FREQUENCY = 1800; +const ABSOLUTE_SILENCE_RMS = 0.0025; +const RELATIVE_SILENCE_RATIO = 0.2; +const MIN_WINDOW_DURATION_SECONDS = 0.08; +const ROOT_NAMES = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B'] as const; + +export interface AudioChordWindow { + barIndex: number; + startBeat: number; + endBeat: number; + startSeconds: number; + endSeconds: number; +} + +export interface AudioChordDetectionRequest { + pcm: Float32Array; + sampleRate: number; + clipStartOffsetSeconds: number; + windows: AudioChordWindow[]; +} + +export interface DetectedAudioChord { + barIndex: number; + startBeat: number; + endBeat: number; + symbol: string; + confidence: number; + rms: number; +} + +export interface AudioChordDetectionProgress { + completedWindows: number; + totalWindows: number; + percent: number; +} + +interface ScoredChord { + symbol: string; + score: number; +} + +const HANN_WINDOW = (() => { + const window = new Float32Array(FFT_SIZE); + for (let i = 0; i < FFT_SIZE; i++) { + window[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / (FFT_SIZE - 1))); + } + return window; +})(); + +function clamp(value: number, min: number, max: number): number { + return Math.max(min, Math.min(max, value)); +} + +function cloneWindow(window: AudioChordWindow): AudioChordWindow { + return { ...window }; +} + +function buildTriadCandidates(chroma: Float64Array): { best: ScoredChord; second: ScoredChord } { + let best: ScoredChord = { symbol: 'N', score: Number.NEGATIVE_INFINITY }; + let second: ScoredChord = { symbol: 'N', score: Number.NEGATIVE_INFINITY }; + + for (let root = 0; root < ROOT_NAMES.length; root++) { + const rootEnergy = chroma[root]; + const minorThird = chroma[(root + 3) % 12]; + const majorThird = chroma[(root + 4) % 12]; + const fifth = chroma[(root + 7) % 12]; + const outsideEnergy = Math.max(0, 1 - (rootEnergy + minorThird + majorThird + fifth)); + + const majorScore = (rootEnergy * 1.2) + (majorThird * 1.0) + (fifth * 0.8) - (outsideEnergy * 0.35) - (minorThird * 0.5); + const minorScore = (rootEnergy * 1.2) + (minorThird * 1.0) + (fifth * 0.8) - (outsideEnergy * 0.35) - (majorThird * 0.5); + + const candidates: ScoredChord[] = [ + { symbol: ROOT_NAMES[root], score: majorScore }, + { symbol: `${ROOT_NAMES[root]}m`, score: minorScore }, + ]; + + for (const candidate of candidates) { + if (candidate.score > best.score) { + second = best; + best = candidate; + } else if (candidate.score > second.score) { + second = candidate; + } + } + } + + return { best, second }; +} + +function analyzeChordWindow( + pcm: Float32Array, + sampleRate: number, + startSeconds: number, + endSeconds: number, +): { symbol: string; confidence: number; rms: number } { + const startSample = Math.max(0, Math.floor(startSeconds * sampleRate)); + const endSample = Math.min(pcm.length, Math.ceil(endSeconds * sampleRate)); + const sampleCount = Math.max(0, endSample - startSample); + + if (sampleCount === 0 || (endSeconds - startSeconds) < MIN_WINDOW_DURATION_SECONDS) { + return { symbol: 'N', confidence: 0, rms: 0 }; + } + + const input = new Float32Array(FFT_SIZE); + const fft = new FFT(FFT_SIZE); + const output = fft.createComplexArray() as number[]; + const chroma = new Float64Array(12); + + let totalEnergy = 0; + for (let i = startSample; i < endSample; i++) { + const value = pcm[i]; + totalEnergy += value * value; + } + const rms = Math.sqrt(totalEnergy / sampleCount); + + const totalHops = Math.max(1, Math.floor(Math.max(0, sampleCount - FFT_SIZE) / HOP_SIZE) + 1); + for (let hop = 0; hop < totalHops; hop++) { + input.fill(0); + const frameOffset = startSample + (hop * HOP_SIZE); + const available = Math.max(0, Math.min(FFT_SIZE, endSample - frameOffset)); + let frameEnergy = 0; + + for (let i = 0; i < available; i++) { + const weighted = pcm[frameOffset + i] * HANN_WINDOW[i]; + input[i] = weighted; + frameEnergy += weighted * weighted; + } + + if (frameEnergy < 1e-7) { + continue; + } + + fft.realTransform(output, input as unknown as number[]); + fft.completeSpectrum(output); + + const earlyFrameWeight = Math.max(0.25, 1.5 - (hop / totalHops)); + for (let bin = 1; bin < FFT_SIZE / 2; bin++) { + const frequency = (bin * sampleRate) / FFT_SIZE; + if (frequency < MIN_ANALYSIS_FREQUENCY || frequency > MAX_ANALYSIS_FREQUENCY) { + continue; + } + + const real = output[2 * bin]; + const imaginary = output[(2 * bin) + 1]; + const magnitude = Math.sqrt((real * real) + (imaginary * imaginary)); + if (magnitude < 1e-6) { + continue; + } + + const midiPitch = 69 + (12 * Math.log2(frequency / 440)); + const roundedPitch = Math.round(midiPitch); + const pitchClass = ((roundedPitch % 12) + 12) % 12; + const centsFromPitchClass = Math.abs(midiPitch - roundedPitch); + const pitchWeight = Math.max(0, 1 - (centsFromPitchClass / 0.5)); + const frequencyWeight = 1 / Math.max(frequency, 80); + chroma[pitchClass] += magnitude * magnitude * pitchWeight * frequencyWeight * earlyFrameWeight; + } + } + + const chromaTotal = chroma.reduce((sum, value) => sum + value, 0); + if (chromaTotal <= 0) { + return { symbol: 'N', confidence: 0, rms }; + } + + for (let i = 0; i < chroma.length; i++) { + chroma[i] /= chromaTotal; + } + + const { best, second } = buildTriadCandidates(chroma); + return { + symbol: best.symbol, + confidence: clamp(best.score - second.score + (best.score * 0.2), 0, 1), + rms, + }; +} + +function smoothDetectedChords(results: DetectedAudioChord[]): DetectedAudioChord[] { + if (results.length < 3) { + return results.map(result => ({ ...result })); + } + + const smoothed = results.map(result => ({ ...result })); + for (let i = 1; i < smoothed.length - 1; i++) { + const previous = smoothed[i - 1]; + const current = smoothed[i]; + const next = smoothed[i + 1]; + if (current.symbol === 'N') { + continue; + } + if (previous.symbol === next.symbol && previous.symbol !== 'N' && current.symbol !== previous.symbol) { + const surroundingConfidence = Math.max(previous.confidence, next.confidence); + if (current.confidence < surroundingConfidence * 0.85) { + current.symbol = previous.symbol; + current.confidence = Math.max(current.confidence, surroundingConfidence * 0.75); + } + } + } + + return smoothed; +} + +export function detectChordsFromAudio( + request: AudioChordDetectionRequest, + onProgress?: (progress: AudioChordDetectionProgress) => void, +): DetectedAudioChord[] { + const windows = request.windows.map(cloneWindow); + if (windows.length === 0) { + return []; + } + + onProgress?.({ + completedWindows: 0, + totalWindows: windows.length, + percent: 0, + }); + + const rawResults: DetectedAudioChord[] = []; + windows.forEach((window, index) => { + const analysis = analyzeChordWindow( + request.pcm, + request.sampleRate, + request.clipStartOffsetSeconds + (window.startSeconds - request.clipStartOffsetSeconds), + request.clipStartOffsetSeconds + (window.endSeconds - request.clipStartOffsetSeconds), + ); + + rawResults.push({ + barIndex: window.barIndex, + startBeat: window.startBeat, + endBeat: window.endBeat, + symbol: analysis.symbol, + confidence: analysis.confidence, + rms: analysis.rms, + }); + + onProgress?.({ + completedWindows: index + 1, + totalWindows: windows.length, + percent: Math.round(((index + 1) / windows.length) * 100), + }); + }); + + const maxRms = rawResults.reduce((max, result) => Math.max(max, result.rms), 0); + const silenceThreshold = Math.max(ABSOLUTE_SILENCE_RMS, maxRms * RELATIVE_SILENCE_RATIO); + const filtered = rawResults.map(result => ( + result.rms < silenceThreshold + ? { ...result, symbol: 'N', confidence: 0 } + : result + )); + + return smoothDetectedChords(filtered); +} diff --git a/src/workers/audioChordDetectionWorker.ts b/src/workers/audioChordDetectionWorker.ts new file mode 100644 index 0000000..da32ae6 --- /dev/null +++ b/src/workers/audioChordDetectionWorker.ts @@ -0,0 +1,24 @@ +import { + detectChordsFromAudio, + type AudioChordDetectionProgress, + type AudioChordDetectionRequest, + type DetectedAudioChord, +} from '../util/audioChordDetectionCore'; + +export type AudioChordDetectionWorkerMessage = + | { type: 'progress'; progress: AudioChordDetectionProgress } + | { type: 'result'; results: DetectedAudioChord[] }; + +type WorkerScopeLike = typeof globalThis & { + onmessage: ((event: MessageEvent) => void) | null; + postMessage: (message: AudioChordDetectionWorkerMessage) => void; +}; + +const workerScope = self as WorkerScopeLike; + +workerScope.onmessage = (event: MessageEvent) => { + const results = detectChordsFromAudio(event.data, progress => { + workerScope.postMessage({ type: 'progress', progress }); + }); + workerScope.postMessage({ type: 'result', results }); +};