feat: implemented chord detection feature for MIDI regions

2026-05-25 23:22:42 -07:00
parent 275cfece99
commit 0f1bad4394
9 changed files with 1274 additions and 74 deletions
@@ -6,32 +6,43 @@ import type {
  ChoiceOption,
  ChordDetectionOptionsResult,
  ConfirmOptions,
+  MidiChordDetectionOptionsResult,
  PromptOptions,
  TimeSigResult,
 } from '../../util/dialogUtil';

 interface DialogInfo {
-  type: 'alert' | 'confirm' | 'prompt' | 'timesig' | 'choice' | 'chord-detection';
+  type: 'alert' | 'confirm' | 'prompt' | 'timesig' | 'choice' | 'chord-detection' | 'midi-chord-detection';
  message: string;
  options?: ConfirmOptions | PromptOptions;
  defaultValue?: string;
  defaultTimeSig?: TimeSigResult;
  choices?: ChoiceOption[];
  defaultChordDetectionOptions?: ChordDetectionOptionsResult;
+  defaultMidiChordDetectionOptions?: MidiChordDetectionOptionsResult;
 }

+const DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS: ChordDetectionOptionsResult = {
+  sensitivity: 50,
+  stability: 50,
+  noChordThreshold: 0,
+  enableSevenths: false,
+};
+
+const DEFAULT_MIDI_CHORD_DETECTION_OPTIONS: MidiChordDetectionOptionsResult = {
+  enableSevenths: false,
+  shortNoteSuppression: 'medium',
+  harmonicFocus: 'favor-sustained-notes',
+};
+
 const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => {
  const [dialog, setDialog] = useState<DialogInfo | null>(null);
  const [isClosing, setIsClosing] = useState(false);
  const [inputValue, setInputValue] = useState('');
  const [timeSigNumerator, setTimeSigNumerator] = useState('');
  const [timeSigDenominator, setTimeSigDenominator] = useState('');
-  const [chordDetectionOptions, setChordDetectionOptions] = useState<ChordDetectionOptionsResult>({
-    sensitivity: 50,
-    stability: 50,
-    noChordThreshold: 0,
-    enableSevenths: false,
-  });
+  const [chordDetectionOptions, setChordDetectionOptions] = useState<ChordDetectionOptionsResult>(DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS);
+  const [midiChordDetectionOptions, setMidiChordDetectionOptions] = useState<MidiChordDetectionOptionsResult>(DEFAULT_MIDI_CHORD_DETECTION_OPTIONS);
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  const resolveRef = useRef<((value: any) => void) | null>(null);
  const pendingValueRef = useRef<unknown>(undefined);
@@ -80,16 +91,22 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
  ): Promise<ChordDetectionOptionsResult | null> => {
    return new Promise<ChordDetectionOptionsResult | null>((resolve) => {
      resolveRef.current = resolve;
-      setChordDetectionOptions(defaultValue ?? {
-        sensitivity: 50,
-        stability: 50,
-        noChordThreshold: 0,
-        enableSevenths: false,
-      });
+      setChordDetectionOptions(defaultValue ?? DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS);
      setDialog({ type: 'chord-detection', message, defaultChordDetectionOptions: defaultValue });
    });
  }, []);

+  const openMidiChordDetectionOptions = useCallback((
+    message: string,
+    defaultValue?: MidiChordDetectionOptionsResult,
+  ): Promise<MidiChordDetectionOptionsResult | null> => {
+    return new Promise<MidiChordDetectionOptionsResult | null>((resolve) => {
+      resolveRef.current = resolve;
+      setMidiChordDetectionOptions(defaultValue ?? DEFAULT_MIDI_CHORD_DETECTION_OPTIONS);
+      setDialog({ type: 'midi-chord-detection', message, defaultMidiChordDetectionOptions: defaultValue });
+    });
+  }, []);
+
  const close = useCallback((value: unknown) => {
    pendingValueRef.current = value;
    setIsClosing(true);
@@ -103,12 +120,8 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
    setInputValue('');
    setTimeSigNumerator('');
    setTimeSigDenominator('');
-    setChordDetectionOptions({
-      sensitivity: 50,
-      stability: 50,
-      noChordThreshold: 0,
-      enableSevenths: false,
-    });
+    setChordDetectionOptions(DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS);
+    setMidiChordDetectionOptions(DEFAULT_MIDI_CHORD_DETECTION_OPTIONS);
    if (resolveRef.current) {
      resolveRef.current(pendingValueRef.current);
      resolveRef.current = null;
@@ -120,7 +133,7 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
  const registered = useRef(false);
  if (!registered.current) {
    registered.current = true;
-    registerDialogFns(openAlert, openConfirm, openPrompt, openTimeSig, openChoice, openChordDetectionOptions);
+    registerDialogFns(openAlert, openConfirm, openPrompt, openTimeSig, openChoice, openChordDetectionOptions, openMidiChordDetectionOptions);
  }

  if (!dialog) {
@@ -132,13 +145,14 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
  const isTimeSig = dialog.type === 'timesig';
  const isChoice = dialog.type === 'choice';
  const isChordDetection = dialog.type === 'chord-detection';
+  const isMidiChordDetection = dialog.type === 'midi-chord-detection';
  const promptOptions = isPrompt ? (dialog.options as PromptOptions | undefined) : undefined;

  const title = isAlert
    ? 'Notice'
    : isTimeSig
      ? 'Time Signature'
-      : isChordDetection
+      : (isChordDetection || isMidiChordDetection)
        ? 'Chord Detection'
        : isPrompt
          ? 'Input'
@@ -167,6 +181,10 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
      close(chordDetectionOptions);
      return;
    }
+    if (isMidiChordDetection) {
+      close(midiChordDetectionOptions);
+      return;
+    }
    close(true);
  };

@@ -177,6 +195,13 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
    setChordDetectionOptions(current => ({ ...current, [key]: value }));
  };

+  const updateMidiChordDetectionOption = <K extends keyof MidiChordDetectionOptionsResult>(
+    key: K,
+    value: MidiChordDetectionOptionsResult[K],
+  ) => {
+    setMidiChordDetectionOptions(current => ({ ...current, [key]: value }));
+  };
+
  return (
    <>
      {children}
@@ -298,6 +323,49 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
                </label>
              </div>
            )}
+            {isMidiChordDetection && (
+              <div className="dialog-chord-detection-form">
+                <div className="dialog-slider-group">
+                  <div className="dialog-slider-header">
+                    <label className="dialog-slider-label" htmlFor="dialog-midi-short-note-suppression">Short Notes</label>
+                  </div>
+                  <select
+                    id="dialog-midi-short-note-suppression"
+                    className="dialog-input"
+                    value={midiChordDetectionOptions.shortNoteSuppression}
+                    onChange={(e) => updateMidiChordDetectionOption('shortNoteSuppression', e.target.value as MidiChordDetectionOptionsResult['shortNoteSuppression'])}
+                    autoFocus
+                  >
+                    <option value="low">Low suppression</option>
+                    <option value="medium">Medium suppression</option>
+                    <option value="high">High suppression</option>
+                  </select>
+                </div>
+                <div className="dialog-slider-group">
+                  <div className="dialog-slider-header">
+                    <label className="dialog-slider-label" htmlFor="dialog-midi-harmonic-focus">Harmonic Focus</label>
+                  </div>
+                  <select
+                    id="dialog-midi-harmonic-focus"
+                    className="dialog-input"
+                    value={midiChordDetectionOptions.harmonicFocus}
+                    onChange={(e) => updateMidiChordDetectionOption('harmonicFocus', e.target.value as MidiChordDetectionOptionsResult['harmonicFocus'])}
+                  >
+                    <option value="balanced">Balanced</option>
+                    <option value="favor-sustained-notes">Favor sustained notes</option>
+                  </select>
+                </div>
+                <label className="dialog-checkbox-row" htmlFor="dialog-midi-enable-sevenths">
+                  <input
+                    id="dialog-midi-enable-sevenths"
+                    type="checkbox"
+                    checked={midiChordDetectionOptions.enableSevenths}
+                    onChange={(e) => updateMidiChordDetectionOption('enableSevenths', e.target.checked)}
+                  />
+                  <span>Chord Detail: Enable sevenths</span>
+                </label>
+              </div>
+            )}
          </div>
          <div className="dialog-footer">
            {!isAlert && (
@@ -323,9 +391,9 @@ const DialogProvider: React.FC<{ children: React.ReactNode }> = ({ children }) =
              <button
                className="dialog-btn dialog-btn-primary"
                onClick={handleConfirm}
-                autoFocus={!isPrompt && !isTimeSig && !isChordDetection}
+                autoFocus={!isPrompt && !isTimeSig && !isChordDetection && !isMidiChordDetection}
              >
-                {isAlert ? 'OK' : ((dialog.options as ConfirmOptions | PromptOptions | undefined)?.confirmLabel ?? (isPrompt || isTimeSig ? 'OK' : isChordDetection ? 'Detect' : 'Yes'))}
+                {isAlert ? 'OK' : ((dialog.options as ConfirmOptions | PromptOptions | undefined)?.confirmLabel ?? (isPrompt || isTimeSig ? 'OK' : (isChordDetection || isMidiChordDetection) ? 'Detect' : 'Yes'))}
              </button>
            )}
          </div>
@@ -20,7 +20,7 @@ import { ReplaceChordRegionsInRangeCommand, UpdateRegionCommand } from '../../co
 import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface';
 import { KGAudioFileStorage } from '../../core/io/KGAudioFileStorage';
 import { getSuitableChords, noteNameToPitchClass } from '../../util/scaleUtil';
-import { showAlert, showChordDetectionOptions } from '../../util/dialogUtil';
+import { showAlert, showChordDetectionOptions, showMidiChordDetectionOptions } from '../../util/dialogUtil';
 import {
  normalizeSpectrogramHeightResolution,
  type SpectrogramHeightResolution,
@@ -32,6 +32,13 @@ import {
  type AudioChordDetectionOptions,
  type DetectedAudioChord,
 } from '../../util/audioChordDetection';
+import {
+  DEFAULT_MIDI_CHORD_DETECTION_OPTIONS,
+  buildMidiChordWindowsForRegion,
+  detectChordsFromMidi,
+  type DetectedMidiChord,
+  type MidiChordDetectionOptions,
+} from '../../util/midiChordDetection';
 import type { AudioChordDetectionWorkerMessage } from '../../workers/audioChordDetectionWorker';
 import type { PianoRollAutomationType } from './pianoRollAutomation';
 import type { SheetMeasureMetric } from './sheetNotationTypes';
@@ -450,22 +457,32 @@ const PianoRoll: React.FC<PianoRollProps> = ({
  };

  const handleDetectChords = useCallback(async () => {
-    if (!audioRegion || !projectName || !trackId) {
-      await showAlert('Open an audio region in spectrogram mode before detecting chords.');
+    if (!audioRegion && !activeRegion) {
+      await showAlert('Open a MIDI or audio region before detecting chords.');
      return;
    }

    const project = KGCore.instance().getCurrentProject();
-    const windows = buildAudioChordWindowsForRegion(project, audioRegion);
-    if (windows.length === 0) {
-      await showAlert('The selected audio region has no audible span to analyze.');
+    const audioWindows = audioRegion ? buildAudioChordWindowsForRegion(project, audioRegion) : null;
+    const midiWindows = !audioRegion && activeRegion ? buildMidiChordWindowsForRegion(project, activeRegion) : null;
+    const chordWindows = audioWindows ?? midiWindows ?? [];
+    if (chordWindows.length === 0) {
+      await showAlert(audioRegion
+        ? 'The selected audio region has no audible span to analyze.'
+        : 'The selected MIDI region has no bars to analyze.'
+      );
      return;
    }

-    const detectionOptions = await showChordDetectionOptions(
-      'Tune chord detection settings before processing.',
-      DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS,
-    );
+    const detectionOptions = audioRegion
+      ? await showChordDetectionOptions(
+        'Tune audio chord detection settings before processing.',
+        DEFAULT_AUDIO_CHORD_DETECTION_OPTIONS,
+      )
+      : await showMidiChordDetectionOptions(
+        'Tune MIDI chord detection settings before processing.',
+        DEFAULT_MIDI_CHORD_DETECTION_OPTIONS,
+      );
    if (!detectionOptions) {
      return;
    }
@@ -475,48 +492,64 @@ const PianoRoll: React.FC<PianoRollProps> = ({
    let worker: Worker | null = null;

    try {
-      let audioBuffer = KGAudioInterface.instance().getAudioBuffer(trackId, audioRegion.getAudioFileId());
-      if (!audioBuffer) {
-        const rawBuffer = await KGAudioFileStorage.loadAudioFile(projectName, audioRegion.getAudioFileId());
-        const actx = Tone.getContext().rawContext as AudioContext;
-        audioBuffer = await actx.decodeAudioData(rawBuffer);
-      }
-
-      const monoPcm = new Float32Array(audioBuffer.length);
-      for (let channelIndex = 0; channelIndex < audioBuffer.numberOfChannels; channelIndex++) {
-        const channelData = audioBuffer.getChannelData(channelIndex);
-        for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) {
-          monoPcm[sampleIndex] += channelData[sampleIndex] / audioBuffer.numberOfChannels;
+      let detectedChords: DetectedAudioChord[] | DetectedMidiChord[];
+      if (audioRegion) {
+        if (!projectName || !trackId) {
+          await showAlert('Open an audio region in spectrogram mode before detecting chords.');
+          return;
        }
-      }

-      const request: AudioChordDetectionRequest = {
-        pcm: monoPcm,
-        sampleRate: audioBuffer.sampleRate,
-        clipStartOffsetSeconds: audioRegion.getClipStartOffsetSeconds(),
-        windows,
-        options: detectionOptions as AudioChordDetectionOptions,
-      };
+        let audioBuffer = KGAudioInterface.instance().getAudioBuffer(trackId, audioRegion.getAudioFileId());
+        if (!audioBuffer) {
+          const rawBuffer = await KGAudioFileStorage.loadAudioFile(projectName, audioRegion.getAudioFileId());
+          const actx = Tone.getContext().rawContext as AudioContext;
+          audioBuffer = await actx.decodeAudioData(rawBuffer);
+        }

-      const detectedChords = await new Promise<DetectedAudioChord[]>((resolve, reject) => {
-        worker = new Worker(
-          new URL('../../workers/audioChordDetectionWorker.ts', import.meta.url),
-          { type: 'module' },
-        );
-
-        worker.onmessage = (event: MessageEvent<AudioChordDetectionWorkerMessage>) => {
-          if (event.data.type === 'progress') {
-            setDetectChordProgressPercent(event.data.progress.percent);
-            return;
+        const monoPcm = new Float32Array(audioBuffer.length);
+        for (let channelIndex = 0; channelIndex < audioBuffer.numberOfChannels; channelIndex++) {
+          const channelData = audioBuffer.getChannelData(channelIndex);
+          for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) {
+            monoPcm[sampleIndex] += channelData[sampleIndex] / audioBuffer.numberOfChannels;
          }
+        }

-          resolve(event.data.results);
+        const request: AudioChordDetectionRequest = {
+          pcm: monoPcm,
+          sampleRate: audioBuffer.sampleRate,
+          clipStartOffsetSeconds: audioRegion.getClipStartOffsetSeconds(),
+          windows: audioWindows ?? [],
+          options: detectionOptions as AudioChordDetectionOptions,
        };
-        worker.onerror = () => {
-          reject(new Error('Chord detection worker failed.'));
-        };
-        worker.postMessage(request, [request.pcm.buffer]);
-      });
+
+        detectedChords = await new Promise<DetectedAudioChord[]>((resolve, reject) => {
+          worker = new Worker(
+            new URL('../../workers/audioChordDetectionWorker.ts', import.meta.url),
+            { type: 'module' },
+          );
+
+          worker.onmessage = (event: MessageEvent<AudioChordDetectionWorkerMessage>) => {
+            if (event.data.type === 'progress') {
+              setDetectChordProgressPercent(event.data.progress.percent);
+              return;
+            }
+
+            resolve(event.data.results);
+          };
+          worker.onerror = () => {
+            reject(new Error('Chord detection worker failed.'));
+          };
+          worker.postMessage(request, [request.pcm.buffer]);
+        });
+      } else {
+        setDetectChordProgressPercent(100);
+        detectedChords = detectChordsFromMidi({
+          project,
+          region: activeRegion as KGMidiRegion,
+          windows: midiWindows ?? [],
+          options: detectionOptions as MidiChordDetectionOptions,
+        });
+      }

      const replacements = detectedChords
        .filter(result => result.symbol !== 'N' && result.endBeat > result.startBeat)
@@ -526,15 +559,18 @@ const PianoRoll: React.FC<PianoRollProps> = ({
          symbol: result.symbol,
        }));

-      const spanStartBeat = windows[0].startBeat;
-      const spanEndBeat = windows[windows.length - 1].endBeat;
+      const spanStartBeat = chordWindows[0].startBeat;
+      const spanEndBeat = chordWindows[chordWindows.length - 1].endBeat;
      KGCore.instance().executeCommand(
        new ReplaceChordRegionsInRangeCommand(spanStartBeat, spanEndBeat, replacements),
      );
      refreshProjectState();
    } catch (error) {
      console.error('Error detecting chords:', error);
-      await showAlert('Failed to detect chords from this audio region.');
+      await showAlert(audioRegion
+        ? 'Failed to detect chords from this audio region.'
+        : 'Failed to detect chords from this MIDI region.'
+      );
    } finally {
      if (worker) {
        worker.terminate();
@@ -542,7 +578,7 @@ const PianoRoll: React.FC<PianoRollProps> = ({
      setIsDetectingChords(false);
      setDetectChordProgressPercent(0);
    }
-  }, [audioRegion, projectName, refreshProjectState, trackId]);
+  }, [activeRegion, audioRegion, projectName, refreshProjectState, trackId]);

  // Handle title click to rename the region
  const handleTitleClick = () => {
@@ -137,6 +137,24 @@ describe('PianoRollToolbar', () => {
    expect(onDetectChords).toHaveBeenCalledTimes(1);
  });

+  it('shows the detect chords action in midi mode and triggers it', () => {
+    const onDetectChords = vi.fn();
+
+    render(
+      <PianoRollToolbar
+        {...baseProps}
+        mode="midi-edit"
+        showAutomationControls={false}
+        onDetectChords={onDetectChords}
+      />
+    );
+
+    fireEvent.click(screen.getByTitle('More options'));
+    fireEvent.click(screen.getByText('Detect chords...'));
+
+    expect(onDetectChords).toHaveBeenCalledTimes(1);
+  });
+
  it('disables the detect chords action while detection is running', () => {
    const onDetectChords = vi.fn();

@@ -89,6 +89,7 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
 }) => {
  const showMidiControls = mode !== 'spectrogram' && !sheetMusicViewEnabled;   // midi-edit and hybrid
  const showSpecControls = !sheetMusicViewEnabled && (mode === 'spectrogram' || mode === 'hybrid');
+  const showDetectChordMenu = !sheetMusicViewEnabled && !!onDetectChords;

  const [showZoomSlider, setShowZoomSlider] = React.useState(false);
  const zoomSliderRef = React.useRef<HTMLDivElement>(null);
@@ -299,7 +300,7 @@ const PianoRollToolbar: React.FC<PianoRollToolbarProps> = ({
          </div>
        )}

-        {showSpecControls && (
+        {showDetectChordMenu && (
          <div className="quant-dropdown-container" ref={specMenuRef}>
            <button
              className="quant-button"