feat: add audio chord detection for spectrogram regions
This commit is contained in:
Binary file not shown.
@@ -235,6 +235,11 @@
|
||||
left: 0;
|
||||
}
|
||||
|
||||
.piano-roll-toolbar .quant-option.disabled {
|
||||
color: #666;
|
||||
cursor: default;
|
||||
}
|
||||
|
||||
.piano-roll-automation-toolbar-group {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
@@ -408,11 +413,37 @@
|
||||
}
|
||||
|
||||
.spectrogram-loading-label {
|
||||
padding: 6px 10px;
|
||||
background: rgba(0, 0, 0, 0.6);
|
||||
color: #aaa;
|
||||
min-width: 240px;
|
||||
padding: 10px 12px;
|
||||
background: rgba(0, 0, 0, 0.72);
|
||||
color: #ddd;
|
||||
font-size: 11px;
|
||||
border-radius: 3px;
|
||||
border: 1px solid #3a3a3a;
|
||||
border-radius: 6px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.piano-roll-progress-block {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.piano-roll-progress-track {
|
||||
width: 100%;
|
||||
height: 8px;
|
||||
background-color: #1d1d1d;
|
||||
border: 1px solid #3a3a3a;
|
||||
border-radius: 999px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.piano-roll-progress-fill {
|
||||
height: 100%;
|
||||
background: linear-gradient(90deg, #5a9fd4 0%, #7cc2f1 100%);
|
||||
transition: width 0.15s linear;
|
||||
}
|
||||
|
||||
.piano-grid-header {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import React, { useRef, useEffect, useState, useCallback, useLayoutEffect, useMemo } from 'react';
|
||||
import './PianoRoll.css';
|
||||
import * as Tone from 'tone';
|
||||
import { useProjectStore } from '../../stores/projectStore';
|
||||
import { FaGripLines } from 'react-icons/fa';
|
||||
import { KGMidiRegion } from '../../core/region/KGMidiRegion';
|
||||
@@ -15,13 +16,21 @@ import { KGMidiTrack, type InstrumentType } from '../../core/track/KGMidiTrack';
|
||||
import { KGPianoRollState } from '../../core/state/KGPianoRollState';
|
||||
import { ConfigManager } from '../../core/config/ConfigManager';
|
||||
import { beatsToBar } from '../../util/midiUtil';
|
||||
import { UpdateRegionCommand } from '../../core/commands';
|
||||
import { ReplaceChordRegionsInRangeCommand, UpdateRegionCommand } from '../../core/commands';
|
||||
import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
|
||||
import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage';
|
||||
import { getSuitableChords, noteNameToPitchClass } from '../../util/scaleUtil';
|
||||
import { showAlert } from '../../util/dialogUtil';
|
||||
import {
|
||||
normalizeSpectrogramHeightResolution,
|
||||
type SpectrogramHeightResolution,
|
||||
} from '../../util/spectrogramUtil';
|
||||
import {
|
||||
buildAudioChordWindowsForRegion,
|
||||
type AudioChordDetectionRequest,
|
||||
type DetectedAudioChord,
|
||||
} from '../../util/audioChordDetection';
|
||||
import type { AudioChordDetectionWorkerMessage } from '../../workers/audioChordDetectionWorker';
|
||||
import type { PianoRollAutomationType } from './pianoRollAutomation';
|
||||
import type { SheetMeasureMetric } from './sheetNotationTypes';
|
||||
import { getSheetPlayheadPixel, getSheetQuantizationOptions, parseSheetQuantization } from './sheetNotation';
|
||||
@@ -59,7 +68,7 @@ const PianoRoll: React.FC<PianoRollProps> = ({
|
||||
}) => {
|
||||
const isSpectrogram = mode === 'spectrogram';
|
||||
const isHybrid = mode === 'hybrid';
|
||||
const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showKGOnePanel, showEventListPanel, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled, bpm, pianoRollScrollRequest, selectedNoteIds, automationRedrawVersion } = useProjectStore();
|
||||
const { maxBars, tracks, updateTrack, timeSignature, showChatBox, showKGOnePanel, showEventListPanel, showInstrumentSelection, keySignature, selectedMode, setSelectedMode, playheadPosition, isPlaying, autoScrollEnabled, bpm, pianoRollScrollRequest, selectedNoteIds, automationRedrawVersion, refreshProjectState } = useProjectStore();
|
||||
|
||||
// Tool state for piano roll
|
||||
const [activeTool, setActiveTool] = useState<'pointer' | 'pencil'>('pointer');
|
||||
@@ -69,6 +78,8 @@ const PianoRoll: React.FC<PianoRollProps> = ({
|
||||
const [spectrogramPower, setSpectrogramPower] = useState<number>(0.5);
|
||||
const [spectrogramHeightResolution, setSpectrogramHeightResolution] =
|
||||
useState<SpectrogramHeightResolution>(3);
|
||||
const [isDetectingChords, setIsDetectingChords] = useState(false);
|
||||
const [detectChordProgressPercent, setDetectChordProgressPercent] = useState(0);
|
||||
|
||||
// Piano roll zoom (1x–8x); updates --region-grid-beat-width CSS variable
|
||||
const [pianoRollZoom, setPianoRollZoom] = useState<number>(() => KGPianoRollState.instance().getPianoRollZoom());
|
||||
@@ -436,6 +447,92 @@ const PianoRoll: React.FC<PianoRollProps> = ({
|
||||
}
|
||||
};
|
||||
|
||||
const handleDetectChords = useCallback(async () => {
|
||||
if (!audioRegion || !projectName || !trackId) {
|
||||
await showAlert('Open an audio region in spectrogram mode before detecting chords.');
|
||||
return;
|
||||
}
|
||||
|
||||
const project = KGCore.instance().getCurrentProject();
|
||||
const windows = buildAudioChordWindowsForRegion(project, audioRegion);
|
||||
if (windows.length === 0) {
|
||||
await showAlert('The selected audio region has no audible span to analyze.');
|
||||
return;
|
||||
}
|
||||
|
||||
setIsDetectingChords(true);
|
||||
setDetectChordProgressPercent(0);
|
||||
let worker: Worker | null = null;
|
||||
|
||||
try {
|
||||
let audioBuffer = KGAudioInterface.instance().getAudioBuffer(trackId, audioRegion.getAudioFileId());
|
||||
if (!audioBuffer) {
|
||||
const rawBuffer = await KGAudioFileStorage.loadAudioFile(projectName, audioRegion.getAudioFileId());
|
||||
const actx = Tone.getContext().rawContext as AudioContext;
|
||||
audioBuffer = await actx.decodeAudioData(rawBuffer);
|
||||
}
|
||||
|
||||
const monoPcm = new Float32Array(audioBuffer.length);
|
||||
for (let channelIndex = 0; channelIndex < audioBuffer.numberOfChannels; channelIndex++) {
|
||||
const channelData = audioBuffer.getChannelData(channelIndex);
|
||||
for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) {
|
||||
monoPcm[sampleIndex] += channelData[sampleIndex] / audioBuffer.numberOfChannels;
|
||||
}
|
||||
}
|
||||
|
||||
const request: AudioChordDetectionRequest = {
|
||||
pcm: monoPcm,
|
||||
sampleRate: audioBuffer.sampleRate,
|
||||
clipStartOffsetSeconds: audioRegion.getClipStartOffsetSeconds(),
|
||||
windows,
|
||||
};
|
||||
|
||||
const detectedChords = await new Promise<DetectedAudioChord[]>((resolve, reject) => {
|
||||
worker = new Worker(
|
||||
new URL('../../workers/audioChordDetectionWorker.ts', import.meta.url),
|
||||
{ type: 'module' },
|
||||
);
|
||||
|
||||
worker.onmessage = (event: MessageEvent<AudioChordDetectionWorkerMessage>) => {
|
||||
if (event.data.type === 'progress') {
|
||||
setDetectChordProgressPercent(event.data.progress.percent);
|
||||
return;
|
||||
}
|
||||
|
||||
resolve(event.data.results);
|
||||
};
|
||||
worker.onerror = () => {
|
||||
reject(new Error('Chord detection worker failed.'));
|
||||
};
|
||||
worker.postMessage(request, [request.pcm.buffer]);
|
||||
});
|
||||
|
||||
const replacements = detectedChords
|
||||
.filter(result => result.symbol !== 'N' && result.endBeat > result.startBeat)
|
||||
.map(result => ({
|
||||
startBeat: result.startBeat,
|
||||
length: result.endBeat - result.startBeat,
|
||||
symbol: result.symbol,
|
||||
}));
|
||||
|
||||
const spanStartBeat = windows[0].startBeat;
|
||||
const spanEndBeat = windows[windows.length - 1].endBeat;
|
||||
KGCore.instance().executeCommand(
|
||||
new ReplaceChordRegionsInRangeCommand(spanStartBeat, spanEndBeat, replacements),
|
||||
);
|
||||
refreshProjectState();
|
||||
} catch (error) {
|
||||
console.error('Error detecting chords:', error);
|
||||
await showAlert('Failed to detect chords from this audio region.');
|
||||
} finally {
|
||||
if (worker) {
|
||||
worker.terminate();
|
||||
}
|
||||
setIsDetectingChords(false);
|
||||
setDetectChordProgressPercent(0);
|
||||
}
|
||||
}, [audioRegion, projectName, refreshProjectState, trackId]);
|
||||
|
||||
// Handle title click to rename the region
|
||||
const handleTitleClick = () => {
|
||||
// If we were just dragging, don't show the rename dialog
|
||||
@@ -1315,6 +1412,8 @@ const PianoRoll: React.FC<PianoRollProps> = ({
|
||||
automationType={automationType}
|
||||
onAutomationToggle={handleAutomationToggle}
|
||||
onAutomationTypeChange={handleAutomationTypeChange}
|
||||
onDetectChords={handleDetectChords}
|
||||
detectingChords={isDetectingChords}
|
||||
/>
|
||||
|
||||
<NoteAttributeBar selectedNotes={selectedNotes} isSpectrogram={isSpectrogram} activeRegion={activeRegion} />
|
||||
@@ -1351,6 +1450,8 @@ const PianoRoll: React.FC<PianoRollProps> = ({
|
||||
sheetKeySignature={keySignature}
|
||||
sheetInstrument={activeInstrument}
|
||||
onSheetMeasureMetricsChange={handleSheetMeasureMetricsChange}
|
||||
overlayMessage={isDetectingChords ? `Detecting chords… (${detectChordProgressPercent.toString().padStart(2, '0')}% completed)` : null}
|
||||
overlayProgressPercent={isDetectingChords ? detectChordProgressPercent : null}
|
||||
/>
|
||||
|
||||
<div
|
||||
|
||||
@@ -120,6 +120,20 @@ describe('PianoRollContent', () => {
|
||||
expect(screen.queryByTestId('automation-lane')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows an overlay message when one is supplied by the parent panel', () => {
|
||||
render(
|
||||
<PianoRollContent
|
||||
{...baseProps}
|
||||
mode="spectrogram"
|
||||
overlayMessage="Detecting chords…"
|
||||
overlayProgressPercent={42}
|
||||
/>
|
||||
);
|
||||
|
||||
expect(screen.getByText('Detecting chords…')).toBeInTheDocument();
|
||||
expect(screen.getByRole('progressbar')).toHaveAttribute('aria-valuenow', '42');
|
||||
});
|
||||
|
||||
it('renders sheet mode without piano keys or automation lane', () => {
|
||||
render(
|
||||
<PianoRollContent
|
||||
|
||||
@@ -55,6 +55,8 @@ interface PianoRollContentProps {
|
||||
sheetKeySignature?: KeySignature;
|
||||
sheetInstrument?: InstrumentType;
|
||||
onSheetMeasureMetricsChange?: (metrics: SheetMeasureMetric[]) => void;
|
||||
overlayMessage?: string | null;
|
||||
overlayProgressPercent?: number | null;
|
||||
}
|
||||
|
||||
const PianoRollContent: React.FC<PianoRollContentProps> = ({
|
||||
@@ -89,6 +91,8 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
|
||||
sheetKeySignature = 'C major',
|
||||
sheetInstrument = 'acoustic_grand_piano',
|
||||
onSheetMeasureMetricsChange,
|
||||
overlayMessage = null,
|
||||
overlayProgressPercent = null,
|
||||
}) => {
|
||||
const isSpectrogram = mode === 'spectrogram';
|
||||
const showAutomationLane = automationEnabled && !isSpectrogram && !sheetMusicViewEnabled;
|
||||
@@ -321,6 +325,8 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
|
||||
container.scrollTop = 0;
|
||||
}, [noteScrollRef, sheetMusicViewEnabled]);
|
||||
|
||||
const effectiveOverlayMessage = overlayMessage ?? (spectrogramLoading ? 'Computing spectrogram…' : null);
|
||||
|
||||
return (
|
||||
<div className="piano-roll-content-outer">
|
||||
<div
|
||||
@@ -397,10 +403,26 @@ const PianoRollContent: React.FC<PianoRollContentProps> = ({
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{spectrogramLoading && (
|
||||
{effectiveOverlayMessage && (
|
||||
<div className="spectrogram-loading-overlay">
|
||||
<div className="spectrogram-loading-label">
|
||||
Computing spectrogram…
|
||||
{effectiveOverlayMessage}
|
||||
{overlayProgressPercent !== null && (
|
||||
<div className="piano-roll-progress-block">
|
||||
<div
|
||||
className="piano-roll-progress-track"
|
||||
role="progressbar"
|
||||
aria-valuemin={0}
|
||||
aria-valuemax={100}
|
||||
aria-valuenow={Math.max(0, Math.min(100, overlayProgressPercent))}
|
||||
>
|
||||
<div
|
||||
className="piano-roll-progress-fill"
|
||||
style={{ width: `${Math.max(0, Math.min(100, overlayProgressPercent))}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -60,6 +60,7 @@ describe('PianoRollToolbar', () => {
|
||||
onChordGuideChange: vi.fn(),
|
||||
zoom: 1,
|
||||
onZoomChange: vi.fn(),
|
||||
onDetectChords: vi.fn(),
|
||||
};
|
||||
|
||||
it('shows automation controls in midi mode and toggles the lane', () => {
|
||||
@@ -118,6 +119,44 @@ describe('PianoRollToolbar', () => {
|
||||
expect(screen.queryByRole('button', { name: /Pitch Bend/i })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows the detect chords action in spectrogram mode and triggers it', () => {
|
||||
const onDetectChords = vi.fn();
|
||||
|
||||
render(
|
||||
<PianoRollToolbar
|
||||
{...baseProps}
|
||||
mode="spectrogram"
|
||||
showAutomationControls={false}
|
||||
onDetectChords={onDetectChords}
|
||||
/>
|
||||
);
|
||||
|
||||
fireEvent.click(screen.getByTitle('More options'));
|
||||
fireEvent.click(screen.getByText('Detect chords...'));
|
||||
|
||||
expect(onDetectChords).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('disables the detect chords action while detection is running', () => {
|
||||
const onDetectChords = vi.fn();
|
||||
|
||||
render(
|
||||
<PianoRollToolbar
|
||||
{...baseProps}
|
||||
mode="spectrogram"
|
||||
showAutomationControls={false}
|
||||
detectingChords={true}
|
||||
onDetectChords={onDetectChords}
|
||||
/>
|
||||
);
|
||||
|
||||
fireEvent.click(screen.getByTitle('More options'));
|
||||
const detectItem = screen.getByText('Detecting chords...');
|
||||
expect(detectItem).toHaveAttribute('aria-disabled', 'true');
|
||||
fireEvent.click(detectItem);
|
||||
expect(onDetectChords).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('shows only the sheet controls when sheet mode is enabled', () => {
|
||||
render(
|
||||
<PianoRollToolbar
|
||||
|
||||
@@ -48,6 +48,8 @@ interface PianoRollToolbarProps {
|
||||
automationType?: PianoRollAutomationType;
|
||||
onAutomationToggle?: () => void;
|
||||
onAutomationTypeChange?: (value: PianoRollAutomationType) => void;
|
||||
onDetectChords?: () => void | Promise<void>;
|
||||
detectingChords?: boolean;
|
||||
}
|
||||
|
||||
const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
|
||||
@@ -82,6 +84,8 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
|
||||
automationType = 'pitch-bend',
|
||||
onAutomationToggle,
|
||||
onAutomationTypeChange,
|
||||
onDetectChords,
|
||||
detectingChords = false,
|
||||
}) => {
|
||||
const showMidiControls = mode !== 'spectrogram' && !sheetMusicViewEnabled; // midi-edit and hybrid
|
||||
const showSpecControls = !sheetMusicViewEnabled && (mode === 'spectrogram' || mode === 'hybrid');
|
||||
@@ -100,6 +104,20 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
|
||||
return () => document.removeEventListener('mousedown', handleClickOutside);
|
||||
}, [showZoomSlider]);
|
||||
|
||||
const [showSpecMenu, setShowSpecMenu] = React.useState(false);
|
||||
const specMenuRef = React.useRef<HTMLDivElement>(null);
|
||||
|
||||
React.useEffect(() => {
|
||||
if (!showSpecMenu) return;
|
||||
const handleClickOutside = (e: MouseEvent) => {
|
||||
if (specMenuRef.current && !specMenuRef.current.contains(e.target as Node)) {
|
||||
setShowSpecMenu(false);
|
||||
}
|
||||
};
|
||||
document.addEventListener('mousedown', handleClickOutside);
|
||||
return () => document.removeEventListener('mousedown', handleClickOutside);
|
||||
}, [showSpecMenu]);
|
||||
|
||||
return (
|
||||
<div className="piano-roll-toolbar">
|
||||
{showMidiControls && (
|
||||
@@ -280,6 +298,35 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{showSpecControls && (
|
||||
<div className="quant-dropdown-container" ref={specMenuRef}>
|
||||
<button
|
||||
className="quant-button"
|
||||
onClick={() => setShowSpecMenu(!showSpecMenu)}
|
||||
title="More options"
|
||||
>
|
||||
...
|
||||
</button>
|
||||
{showSpecMenu && (
|
||||
<div className="quant-dropdown" style={{ right: 0, left: 'auto', width: 'auto', whiteSpace: 'nowrap' }}>
|
||||
<div
|
||||
className={`quant-option${(!onDetectChords || detectingChords) ? ' disabled' : ''}`}
|
||||
onClick={() => {
|
||||
if (!onDetectChords || detectingChords) {
|
||||
return;
|
||||
}
|
||||
setShowSpecMenu(false);
|
||||
void onDetectChords();
|
||||
}}
|
||||
aria-disabled={!onDetectChords || detectingChords}
|
||||
>
|
||||
{detectingChords ? 'Detecting chords...' : 'Detect chords...'}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -6,6 +6,7 @@ import { KGChordRegion } from '../../region/KGChordRegion';
|
||||
import { CreateChordRegionCommand } from './CreateChordRegionCommand';
|
||||
import { InsertChordRegionAtBeatCommand } from './InsertChordRegionAtBeatCommand';
|
||||
import { MoveGlobalRegionCommand } from './MoveGlobalRegionCommand';
|
||||
import { ReplaceChordRegionsInRangeCommand } from './ReplaceChordRegionsInRangeCommand';
|
||||
import { ResizeGlobalRegionCommand } from './ResizeGlobalRegionCommand';
|
||||
import { UpdateChordRegionCommand } from './UpdateChordRegionCommand';
|
||||
|
||||
@@ -104,4 +105,46 @@ describe('global chord region commands', () => {
|
||||
expect(chordTrack.getRegions()).toHaveLength(1);
|
||||
expect(region.getLength()).toBe(8);
|
||||
});
|
||||
|
||||
it('replaces only the requested chord span and restores the original layout on undo', () => {
|
||||
const chordTrack = getChordTrack();
|
||||
chordTrack.setRegions([
|
||||
new KGChordRegion('left', chordTrack.getId(), chordTrack.getTrackIndex(), 'C', 0, 4),
|
||||
new KGChordRegion('middle', chordTrack.getId(), chordTrack.getTrackIndex(), 'Am', 4, 4),
|
||||
new KGChordRegion('right', chordTrack.getId(), chordTrack.getTrackIndex(), 'F', 8, 4),
|
||||
]);
|
||||
|
||||
const command = new ReplaceChordRegionsInRangeCommand(2, 10, [
|
||||
{ startBeat: 2, length: 2, symbol: 'Dm' },
|
||||
{ startBeat: 4, length: 4, symbol: 'E' },
|
||||
{ startBeat: 8, length: 2, symbol: 'Am' },
|
||||
]);
|
||||
|
||||
command.execute();
|
||||
|
||||
const replacedRegions = getChordTrack().getRegions() as KGChordRegion[];
|
||||
expect(replacedRegions.map(region => ({
|
||||
symbol: region.getSymbol(),
|
||||
start: region.getStartFromBeat(),
|
||||
length: region.getLength(),
|
||||
}))).toEqual([
|
||||
{ symbol: 'C', start: 0, length: 2 },
|
||||
{ symbol: 'Dm', start: 2, length: 2 },
|
||||
{ symbol: 'E', start: 4, length: 4 },
|
||||
{ symbol: 'Am', start: 8, length: 2 },
|
||||
{ symbol: 'F', start: 10, length: 2 },
|
||||
]);
|
||||
|
||||
command.undo();
|
||||
const restoredRegions = getChordTrack().getRegions() as KGChordRegion[];
|
||||
expect(restoredRegions.map(region => ({
|
||||
symbol: region.getSymbol(),
|
||||
start: region.getStartFromBeat(),
|
||||
length: region.getLength(),
|
||||
}))).toEqual([
|
||||
{ symbol: 'C', start: 0, length: 4 },
|
||||
{ symbol: 'Am', start: 4, length: 4 },
|
||||
{ symbol: 'F', start: 8, length: 4 },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
import { KGCommand } from '../KGCommand';
|
||||
import { KGCore } from '../../KGCore';
|
||||
import { GlobalTrackType } from '../../global-track';
|
||||
import { KGChordRegion } from '../../region/KGChordRegion';
|
||||
import { findGlobalTrackByType } from '../../../util/globalTrackUtil';
|
||||
import { generateUniqueId } from '../../../util/miscUtil';
|
||||
|
||||
export interface ChordRegionReplacementData {
|
||||
startBeat: number;
|
||||
length: number;
|
||||
symbol: string;
|
||||
}
|
||||
|
||||
function cloneChordRegion(region: KGChordRegion): KGChordRegion {
|
||||
return new KGChordRegion(
|
||||
region.getId(),
|
||||
region.getTrackId(),
|
||||
region.getTrackIndex(),
|
||||
region.getSymbol(),
|
||||
region.getStartFromBeat(),
|
||||
region.getLength(),
|
||||
);
|
||||
}
|
||||
|
||||
function cloneChordRegions(regions: KGChordRegion[]): KGChordRegion[] {
|
||||
return regions.map(cloneChordRegion);
|
||||
}
|
||||
|
||||
export class ReplaceChordRegionsInRangeCommand extends KGCommand {
|
||||
private readonly rangeStartBeat: number;
|
||||
private readonly rangeEndBeat: number;
|
||||
private readonly replacements: ChordRegionReplacementData[];
|
||||
private originalRegions: KGChordRegion[] | null = null;
|
||||
private nextRegions: KGChordRegion[] | null = null;
|
||||
|
||||
constructor(rangeStartBeat: number, rangeEndBeat: number, replacements: ChordRegionReplacementData[]) {
|
||||
super();
|
||||
this.rangeStartBeat = Math.max(0, rangeStartBeat);
|
||||
this.rangeEndBeat = Math.max(this.rangeStartBeat, rangeEndBeat);
|
||||
this.replacements = replacements.map(replacement => ({
|
||||
startBeat: replacement.startBeat,
|
||||
length: replacement.length,
|
||||
symbol: replacement.symbol,
|
||||
}));
|
||||
}
|
||||
|
||||
execute(): void {
|
||||
const project = KGCore.instance().getCurrentProject();
|
||||
const chordTrack = findGlobalTrackByType(project, GlobalTrackType.Chord);
|
||||
if (!chordTrack) {
|
||||
throw new Error('Chord global track not found');
|
||||
}
|
||||
|
||||
if (this.nextRegions) {
|
||||
chordTrack.setRegions(cloneChordRegions(this.nextRegions));
|
||||
return;
|
||||
}
|
||||
|
||||
const currentRegions = chordTrack.getRegions()
|
||||
.filter((region): region is KGChordRegion => region instanceof KGChordRegion)
|
||||
.sort((left, right) => left.getStartFromBeat() - right.getStartFromBeat());
|
||||
|
||||
this.originalRegions = cloneChordRegions(currentRegions);
|
||||
|
||||
const preservedRegions: KGChordRegion[] = [];
|
||||
for (const region of currentRegions) {
|
||||
const regionStart = region.getStartFromBeat();
|
||||
const regionEnd = regionStart + region.getLength();
|
||||
|
||||
if (regionEnd <= this.rangeStartBeat || regionStart >= this.rangeEndBeat) {
|
||||
preservedRegions.push(cloneChordRegion(region));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (regionStart < this.rangeStartBeat) {
|
||||
preservedRegions.push(new KGChordRegion(
|
||||
region.getId(),
|
||||
region.getTrackId(),
|
||||
region.getTrackIndex(),
|
||||
region.getSymbol(),
|
||||
regionStart,
|
||||
this.rangeStartBeat - regionStart,
|
||||
));
|
||||
}
|
||||
|
||||
if (regionEnd > this.rangeEndBeat) {
|
||||
preservedRegions.push(new KGChordRegion(
|
||||
generateUniqueId('KGChordRegion'),
|
||||
region.getTrackId(),
|
||||
region.getTrackIndex(),
|
||||
region.getSymbol(),
|
||||
this.rangeEndBeat,
|
||||
regionEnd - this.rangeEndBeat,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
const replacementRegions = this.replacements
|
||||
.filter(replacement => replacement.length > 0 && replacement.symbol.trim() !== '')
|
||||
.map(replacement => new KGChordRegion(
|
||||
generateUniqueId('KGChordRegion'),
|
||||
chordTrack.getId(),
|
||||
chordTrack.getTrackIndex(),
|
||||
replacement.symbol,
|
||||
replacement.startBeat,
|
||||
replacement.length,
|
||||
));
|
||||
|
||||
this.nextRegions = [...preservedRegions, ...replacementRegions]
|
||||
.sort((left, right) => left.getStartFromBeat() - right.getStartFromBeat());
|
||||
|
||||
chordTrack.setRegions(cloneChordRegions(this.nextRegions));
|
||||
}
|
||||
|
||||
undo(): void {
|
||||
if (!this.originalRegions) {
|
||||
throw new Error('Cannot undo chord replacement without original regions');
|
||||
}
|
||||
|
||||
const project = KGCore.instance().getCurrentProject();
|
||||
const chordTrack = findGlobalTrackByType(project, GlobalTrackType.Chord);
|
||||
if (!chordTrack) {
|
||||
throw new Error('Chord global track not found during undo');
|
||||
}
|
||||
|
||||
chordTrack.setRegions(cloneChordRegions(this.originalRegions));
|
||||
}
|
||||
|
||||
getDescription(): string {
|
||||
return 'Replace chord regions in range';
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,10 @@ export { MergeMidiRegionsCommand } from './region/MergeMidiRegionsCommand';
|
||||
export { CreateGlobalMarkerRegionCommand } from './global-region/CreateGlobalMarkerRegionCommand';
|
||||
export { CreateChordRegionCommand } from './global-region/CreateChordRegionCommand';
|
||||
export { InsertChordRegionAtBeatCommand } from './global-region/InsertChordRegionAtBeatCommand';
|
||||
export {
|
||||
ReplaceChordRegionsInRangeCommand,
|
||||
type ChordRegionReplacementData,
|
||||
} from './global-region/ReplaceChordRegionsInRangeCommand';
|
||||
export { CreateKeySignatureRegionCommand } from './global-region/CreateKeySignatureRegionCommand';
|
||||
export { CreateTempoRegionCommand } from './global-region/CreateTempoRegionCommand';
|
||||
export { MoveGlobalRegionCommand } from './global-region/MoveGlobalRegionCommand';
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
import { execFileSync, spawnSync } from 'node:child_process';
|
||||
import path from 'node:path';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { detectChordsFromAudio } from '../../util/audioChordDetectionCore';
|
||||
|
||||
const FIXTURE_PATH = path.resolve(process.cwd(), 'public/test-data/chord-progression-01.mp3');
|
||||
const BAR_DURATION_SECONDS = 2;
|
||||
const ffmpegAvailable = spawnSync('ffmpeg', ['-version'], { stdio: 'ignore' }).status === 0;
|
||||
const runIfFfmpeg = ffmpegAvailable ? it : it.skip;
|
||||
|
||||
function decodeMp3ToMonoPcm(path: string): { sampleRate: number; pcm: Float32Array } {
|
||||
const wav = execFileSync(
|
||||
'ffmpeg',
|
||||
['-v', 'error', '-i', path, '-ac', '1', '-f', 'wav', 'pipe:1'],
|
||||
{ maxBuffer: 64 * 1024 * 1024 },
|
||||
);
|
||||
|
||||
const readString = (offset: number, length: number) => wav.toString('ascii', offset, offset + length);
|
||||
if (readString(0, 4) !== 'RIFF' || readString(8, 4) !== 'WAVE') {
|
||||
throw new Error('ffmpeg did not return a RIFF/WAVE stream');
|
||||
}
|
||||
|
||||
let offset = 12;
|
||||
let sampleRate = 44100;
|
||||
let pcmData = Buffer.alloc(0);
|
||||
|
||||
while (offset + 8 <= wav.length) {
|
||||
const chunkId = readString(offset, 4);
|
||||
const chunkSize = wav.readUInt32LE(offset + 4);
|
||||
const chunkStart = offset + 8;
|
||||
|
||||
if (chunkId === 'fmt ') {
|
||||
sampleRate = wav.readUInt32LE(chunkStart + 4);
|
||||
} else if (chunkId === 'data') {
|
||||
pcmData = wav.subarray(chunkStart, chunkStart + chunkSize);
|
||||
break;
|
||||
}
|
||||
|
||||
offset = chunkStart + chunkSize + (chunkSize % 2);
|
||||
}
|
||||
|
||||
const pcm = new Float32Array(pcmData.length / 2);
|
||||
for (let sampleIndex = 0; sampleIndex < pcm.length; sampleIndex++) {
|
||||
pcm[sampleIndex] = pcmData.readInt16LE(sampleIndex * 2) / 32768;
|
||||
}
|
||||
|
||||
return { sampleRate, pcm };
|
||||
}
|
||||
|
||||
describe('audio chord detection fixture', () => {
|
||||
runIfFfmpeg('detects the expected bar-locked progression from the mp3 fixture', () => {
|
||||
const { sampleRate, pcm } = decodeMp3ToMonoPcm(FIXTURE_PATH);
|
||||
const windows = Array.from({ length: 9 }, (_, barIndex) => ({
|
||||
barIndex,
|
||||
startBeat: barIndex * 4,
|
||||
endBeat: (barIndex + 1) * 4,
|
||||
startSeconds: barIndex * BAR_DURATION_SECONDS,
|
||||
endSeconds: (barIndex + 1) * BAR_DURATION_SECONDS,
|
||||
}));
|
||||
|
||||
const results = detectChordsFromAudio({
|
||||
pcm,
|
||||
sampleRate,
|
||||
clipStartOffsetSeconds: 0,
|
||||
windows,
|
||||
});
|
||||
|
||||
expect(results.map(result => result.symbol)).toEqual([
|
||||
'Am',
|
||||
'F',
|
||||
'Dm',
|
||||
'E',
|
||||
'Am',
|
||||
'C',
|
||||
'Dm',
|
||||
'E',
|
||||
'N',
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,75 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { detectChordsFromAudio, type AudioChordDetectionRequest } from './audioChordDetectionCore';
|
||||
|
||||
const SAMPLE_RATE = 44100;
|
||||
|
||||
function createSineChordPcm(frequencies: number[], durationSeconds: number): Float32Array {
|
||||
const sampleCount = Math.floor(durationSeconds * SAMPLE_RATE);
|
||||
const pcm = new Float32Array(sampleCount);
|
||||
|
||||
for (let sampleIndex = 0; sampleIndex < sampleCount; sampleIndex++) {
|
||||
const time = sampleIndex / SAMPLE_RATE;
|
||||
let sample = 0;
|
||||
for (const frequency of frequencies) {
|
||||
sample += Math.sin(2 * Math.PI * frequency * time);
|
||||
}
|
||||
pcm[sampleIndex] = (sample / Math.max(1, frequencies.length)) * 0.35;
|
||||
}
|
||||
|
||||
return pcm;
|
||||
}
|
||||
|
||||
function createRequest(windows: AudioChordDetectionRequest['windows'], pcm: Float32Array): AudioChordDetectionRequest {
|
||||
return {
|
||||
pcm,
|
||||
sampleRate: SAMPLE_RATE,
|
||||
clipStartOffsetSeconds: 0,
|
||||
windows,
|
||||
};
|
||||
}
|
||||
|
||||
describe('audio chord detection', () => {
|
||||
it('detects a major triad from synthetic audio', () => {
|
||||
const pcm = createSineChordPcm([261.63, 329.63, 392.0], 2);
|
||||
const [result] = detectChordsFromAudio(createRequest([
|
||||
{ barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 },
|
||||
], pcm));
|
||||
|
||||
expect(result.symbol).toBe('C');
|
||||
expect(result.confidence).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('detects a minor triad from synthetic audio', () => {
|
||||
const pcm = createSineChordPcm([220.0, 261.63, 329.63], 2);
|
||||
const [result] = detectChordsFromAudio(createRequest([
|
||||
{ barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 },
|
||||
], pcm));
|
||||
|
||||
expect(result.symbol).toBe('Am');
|
||||
});
|
||||
|
||||
it('marks silent analysis windows as no chord', () => {
|
||||
const pcm = new Float32Array(SAMPLE_RATE * 2);
|
||||
const [result] = detectChordsFromAudio(createRequest([
|
||||
{ barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 },
|
||||
], pcm));
|
||||
|
||||
expect(result.symbol).toBe('N');
|
||||
expect(result.confidence).toBe(0);
|
||||
});
|
||||
|
||||
it('keeps neighboring synthetic bars stable', () => {
|
||||
const barA = createSineChordPcm([220.0, 261.63, 329.63], 2);
|
||||
const barB = createSineChordPcm([220.0, 261.63, 329.63], 2);
|
||||
const pcm = new Float32Array(barA.length + barB.length);
|
||||
pcm.set(barA, 0);
|
||||
pcm.set(barB, barA.length);
|
||||
|
||||
const results = detectChordsFromAudio(createRequest([
|
||||
{ barIndex: 0, startBeat: 0, endBeat: 4, startSeconds: 0, endSeconds: 2 },
|
||||
{ barIndex: 1, startBeat: 4, endBeat: 8, startSeconds: 2, endSeconds: 4 },
|
||||
], pcm));
|
||||
|
||||
expect(results.map(result => result.symbol)).toEqual(['Am', 'Am']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
import { KGProject } from '../core/KGProject';
|
||||
import { KGAudioRegion } from '../core/region/KGAudioRegion';
|
||||
import { beatRangeToSeconds, getAudioRegionDisplayLengthBeats } from './globalTrackUtil';
|
||||
|
||||
export {
|
||||
detectChordsFromAudio,
|
||||
type AudioChordDetectionRequest,
|
||||
type AudioChordWindow,
|
||||
type DetectedAudioChord,
|
||||
} from './audioChordDetectionCore';
|
||||
|
||||
import type { AudioChordWindow } from './audioChordDetectionCore';
|
||||
|
||||
export function buildAudioChordWindowsForRegion(
|
||||
project: KGProject,
|
||||
audioRegion: KGAudioRegion,
|
||||
): AudioChordWindow[] {
|
||||
const regionStartBeat = audioRegion.getStartFromBeat();
|
||||
const visibleLengthBeats = getAudioRegionDisplayLengthBeats(project, audioRegion);
|
||||
if (visibleLengthBeats <= 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const regionEndBeat = regionStartBeat + visibleLengthBeats;
|
||||
const beatsPerBar = project.getTimeSignature().numerator;
|
||||
const startBarIndex = Math.floor(regionStartBeat / beatsPerBar);
|
||||
const lastBeatExclusive = regionEndBeat - 1e-9;
|
||||
const endBarIndexExclusive = Math.max(
|
||||
startBarIndex + 1,
|
||||
Math.ceil(Math.max(regionStartBeat, lastBeatExclusive) / beatsPerBar),
|
||||
);
|
||||
|
||||
const windows: AudioChordWindow[] = [];
|
||||
for (let barIndex = startBarIndex; barIndex < endBarIndexExclusive; barIndex++) {
|
||||
const barStartBeat = barIndex * beatsPerBar;
|
||||
const barEndBeat = barStartBeat + beatsPerBar;
|
||||
const overlapStartBeat = Math.max(regionStartBeat, barStartBeat);
|
||||
const overlapEndBeat = Math.min(regionEndBeat, barEndBeat);
|
||||
if (overlapEndBeat <= overlapStartBeat) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const startSeconds = audioRegion.getClipStartOffsetSeconds() + beatRangeToSeconds(project, regionStartBeat, overlapStartBeat);
|
||||
const endSeconds = audioRegion.getClipStartOffsetSeconds() + beatRangeToSeconds(project, regionStartBeat, overlapEndBeat);
|
||||
windows.push({
|
||||
barIndex,
|
||||
startBeat: overlapStartBeat,
|
||||
endBeat: overlapEndBeat,
|
||||
startSeconds,
|
||||
endSeconds,
|
||||
});
|
||||
}
|
||||
|
||||
return windows;
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
import FFT from 'fft.js';
|
||||
|
||||
const FFT_SIZE = 4096;
|
||||
const HOP_SIZE = 512;
|
||||
const MIN_ANALYSIS_FREQUENCY = 55;
|
||||
const MAX_ANALYSIS_FREQUENCY = 1800;
|
||||
const ABSOLUTE_SILENCE_RMS = 0.0025;
|
||||
const RELATIVE_SILENCE_RATIO = 0.2;
|
||||
const MIN_WINDOW_DURATION_SECONDS = 0.08;
|
||||
const ROOT_NAMES = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B'] as const;
|
||||
|
||||
export interface AudioChordWindow {
|
||||
barIndex: number;
|
||||
startBeat: number;
|
||||
endBeat: number;
|
||||
startSeconds: number;
|
||||
endSeconds: number;
|
||||
}
|
||||
|
||||
export interface AudioChordDetectionRequest {
|
||||
pcm: Float32Array;
|
||||
sampleRate: number;
|
||||
clipStartOffsetSeconds: number;
|
||||
windows: AudioChordWindow[];
|
||||
}
|
||||
|
||||
export interface DetectedAudioChord {
|
||||
barIndex: number;
|
||||
startBeat: number;
|
||||
endBeat: number;
|
||||
symbol: string;
|
||||
confidence: number;
|
||||
rms: number;
|
||||
}
|
||||
|
||||
export interface AudioChordDetectionProgress {
|
||||
completedWindows: number;
|
||||
totalWindows: number;
|
||||
percent: number;
|
||||
}
|
||||
|
||||
interface ScoredChord {
|
||||
symbol: string;
|
||||
score: number;
|
||||
}
|
||||
|
||||
const HANN_WINDOW = (() => {
|
||||
const window = new Float32Array(FFT_SIZE);
|
||||
for (let i = 0; i < FFT_SIZE; i++) {
|
||||
window[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / (FFT_SIZE - 1)));
|
||||
}
|
||||
return window;
|
||||
})();
|
||||
|
||||
function clamp(value: number, min: number, max: number): number {
|
||||
return Math.max(min, Math.min(max, value));
|
||||
}
|
||||
|
||||
function cloneWindow(window: AudioChordWindow): AudioChordWindow {
|
||||
return { ...window };
|
||||
}
|
||||
|
||||
function buildTriadCandidates(chroma: Float64Array): { best: ScoredChord; second: ScoredChord } {
|
||||
let best: ScoredChord = { symbol: 'N', score: Number.NEGATIVE_INFINITY };
|
||||
let second: ScoredChord = { symbol: 'N', score: Number.NEGATIVE_INFINITY };
|
||||
|
||||
for (let root = 0; root < ROOT_NAMES.length; root++) {
|
||||
const rootEnergy = chroma[root];
|
||||
const minorThird = chroma[(root + 3) % 12];
|
||||
const majorThird = chroma[(root + 4) % 12];
|
||||
const fifth = chroma[(root + 7) % 12];
|
||||
const outsideEnergy = Math.max(0, 1 - (rootEnergy + minorThird + majorThird + fifth));
|
||||
|
||||
const majorScore = (rootEnergy * 1.2) + (majorThird * 1.0) + (fifth * 0.8) - (outsideEnergy * 0.35) - (minorThird * 0.5);
|
||||
const minorScore = (rootEnergy * 1.2) + (minorThird * 1.0) + (fifth * 0.8) - (outsideEnergy * 0.35) - (majorThird * 0.5);
|
||||
|
||||
const candidates: ScoredChord[] = [
|
||||
{ symbol: ROOT_NAMES[root], score: majorScore },
|
||||
{ symbol: `${ROOT_NAMES[root]}m`, score: minorScore },
|
||||
];
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.score > best.score) {
|
||||
second = best;
|
||||
best = candidate;
|
||||
} else if (candidate.score > second.score) {
|
||||
second = candidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { best, second };
|
||||
}
|
||||
|
||||
function analyzeChordWindow(
|
||||
pcm: Float32Array,
|
||||
sampleRate: number,
|
||||
startSeconds: number,
|
||||
endSeconds: number,
|
||||
): { symbol: string; confidence: number; rms: number } {
|
||||
const startSample = Math.max(0, Math.floor(startSeconds * sampleRate));
|
||||
const endSample = Math.min(pcm.length, Math.ceil(endSeconds * sampleRate));
|
||||
const sampleCount = Math.max(0, endSample - startSample);
|
||||
|
||||
if (sampleCount === 0 || (endSeconds - startSeconds) < MIN_WINDOW_DURATION_SECONDS) {
|
||||
return { symbol: 'N', confidence: 0, rms: 0 };
|
||||
}
|
||||
|
||||
const input = new Float32Array(FFT_SIZE);
|
||||
const fft = new FFT(FFT_SIZE);
|
||||
const output = fft.createComplexArray() as number[];
|
||||
const chroma = new Float64Array(12);
|
||||
|
||||
let totalEnergy = 0;
|
||||
for (let i = startSample; i < endSample; i++) {
|
||||
const value = pcm[i];
|
||||
totalEnergy += value * value;
|
||||
}
|
||||
const rms = Math.sqrt(totalEnergy / sampleCount);
|
||||
|
||||
const totalHops = Math.max(1, Math.floor(Math.max(0, sampleCount - FFT_SIZE) / HOP_SIZE) + 1);
|
||||
for (let hop = 0; hop < totalHops; hop++) {
|
||||
input.fill(0);
|
||||
const frameOffset = startSample + (hop * HOP_SIZE);
|
||||
const available = Math.max(0, Math.min(FFT_SIZE, endSample - frameOffset));
|
||||
let frameEnergy = 0;
|
||||
|
||||
for (let i = 0; i < available; i++) {
|
||||
const weighted = pcm[frameOffset + i] * HANN_WINDOW[i];
|
||||
input[i] = weighted;
|
||||
frameEnergy += weighted * weighted;
|
||||
}
|
||||
|
||||
if (frameEnergy < 1e-7) {
|
||||
continue;
|
||||
}
|
||||
|
||||
fft.realTransform(output, input as unknown as number[]);
|
||||
fft.completeSpectrum(output);
|
||||
|
||||
const earlyFrameWeight = Math.max(0.25, 1.5 - (hop / totalHops));
|
||||
for (let bin = 1; bin < FFT_SIZE / 2; bin++) {
|
||||
const frequency = (bin * sampleRate) / FFT_SIZE;
|
||||
if (frequency < MIN_ANALYSIS_FREQUENCY || frequency > MAX_ANALYSIS_FREQUENCY) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const real = output[2 * bin];
|
||||
const imaginary = output[(2 * bin) + 1];
|
||||
const magnitude = Math.sqrt((real * real) + (imaginary * imaginary));
|
||||
if (magnitude < 1e-6) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const midiPitch = 69 + (12 * Math.log2(frequency / 440));
|
||||
const roundedPitch = Math.round(midiPitch);
|
||||
const pitchClass = ((roundedPitch % 12) + 12) % 12;
|
||||
const centsFromPitchClass = Math.abs(midiPitch - roundedPitch);
|
||||
const pitchWeight = Math.max(0, 1 - (centsFromPitchClass / 0.5));
|
||||
const frequencyWeight = 1 / Math.max(frequency, 80);
|
||||
chroma[pitchClass] += magnitude * magnitude * pitchWeight * frequencyWeight * earlyFrameWeight;
|
||||
}
|
||||
}
|
||||
|
||||
const chromaTotal = chroma.reduce((sum, value) => sum + value, 0);
|
||||
if (chromaTotal <= 0) {
|
||||
return { symbol: 'N', confidence: 0, rms };
|
||||
}
|
||||
|
||||
for (let i = 0; i < chroma.length; i++) {
|
||||
chroma[i] /= chromaTotal;
|
||||
}
|
||||
|
||||
const { best, second } = buildTriadCandidates(chroma);
|
||||
return {
|
||||
symbol: best.symbol,
|
||||
confidence: clamp(best.score - second.score + (best.score * 0.2), 0, 1),
|
||||
rms,
|
||||
};
|
||||
}
|
||||
|
||||
function smoothDetectedChords(results: DetectedAudioChord[]): DetectedAudioChord[] {
|
||||
if (results.length < 3) {
|
||||
return results.map(result => ({ ...result }));
|
||||
}
|
||||
|
||||
const smoothed = results.map(result => ({ ...result }));
|
||||
for (let i = 1; i < smoothed.length - 1; i++) {
|
||||
const previous = smoothed[i - 1];
|
||||
const current = smoothed[i];
|
||||
const next = smoothed[i + 1];
|
||||
if (current.symbol === 'N') {
|
||||
continue;
|
||||
}
|
||||
if (previous.symbol === next.symbol && previous.symbol !== 'N' && current.symbol !== previous.symbol) {
|
||||
const surroundingConfidence = Math.max(previous.confidence, next.confidence);
|
||||
if (current.confidence < surroundingConfidence * 0.85) {
|
||||
current.symbol = previous.symbol;
|
||||
current.confidence = Math.max(current.confidence, surroundingConfidence * 0.75);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return smoothed;
|
||||
}
|
||||
|
||||
export function detectChordsFromAudio(
|
||||
request: AudioChordDetectionRequest,
|
||||
onProgress?: (progress: AudioChordDetectionProgress) => void,
|
||||
): DetectedAudioChord[] {
|
||||
const windows = request.windows.map(cloneWindow);
|
||||
if (windows.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
onProgress?.({
|
||||
completedWindows: 0,
|
||||
totalWindows: windows.length,
|
||||
percent: 0,
|
||||
});
|
||||
|
||||
const rawResults: DetectedAudioChord[] = [];
|
||||
windows.forEach((window, index) => {
|
||||
const analysis = analyzeChordWindow(
|
||||
request.pcm,
|
||||
request.sampleRate,
|
||||
request.clipStartOffsetSeconds + (window.startSeconds - request.clipStartOffsetSeconds),
|
||||
request.clipStartOffsetSeconds + (window.endSeconds - request.clipStartOffsetSeconds),
|
||||
);
|
||||
|
||||
rawResults.push({
|
||||
barIndex: window.barIndex,
|
||||
startBeat: window.startBeat,
|
||||
endBeat: window.endBeat,
|
||||
symbol: analysis.symbol,
|
||||
confidence: analysis.confidence,
|
||||
rms: analysis.rms,
|
||||
});
|
||||
|
||||
onProgress?.({
|
||||
completedWindows: index + 1,
|
||||
totalWindows: windows.length,
|
||||
percent: Math.round(((index + 1) / windows.length) * 100),
|
||||
});
|
||||
});
|
||||
|
||||
const maxRms = rawResults.reduce((max, result) => Math.max(max, result.rms), 0);
|
||||
const silenceThreshold = Math.max(ABSOLUTE_SILENCE_RMS, maxRms * RELATIVE_SILENCE_RATIO);
|
||||
const filtered = rawResults.map(result => (
|
||||
result.rms < silenceThreshold
|
||||
? { ...result, symbol: 'N', confidence: 0 }
|
||||
: result
|
||||
));
|
||||
|
||||
return smoothDetectedChords(filtered);
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import {
|
||||
detectChordsFromAudio,
|
||||
type AudioChordDetectionProgress,
|
||||
type AudioChordDetectionRequest,
|
||||
type DetectedAudioChord,
|
||||
} from '../util/audioChordDetectionCore';
|
||||
|
||||
export type AudioChordDetectionWorkerMessage =
|
||||
| { type: 'progress'; progress: AudioChordDetectionProgress }
|
||||
| { type: 'result'; results: DetectedAudioChord[] };
|
||||
|
||||
type WorkerScopeLike = typeof globalThis & {
|
||||
onmessage: ((event: MessageEvent<AudioChordDetectionRequest>) => void) | null;
|
||||
postMessage: (message: AudioChordDetectionWorkerMessage) => void;
|
||||
};
|
||||
|
||||
const workerScope = self as WorkerScopeLike;
|
||||
|
||||
workerScope.onmessage = (event: MessageEvent<AudioChordDetectionRequest>) => {
|
||||
const results = detectChordsFromAudio(event.data, progress => {
|
||||
workerScope.postMessage({ type: 'progress', progress });
|
||||
});
|
||||
workerScope.postMessage({ type: 'result', results });
|
||||
};
|
||||
Reference in New Issue
Block a user